Speed up image file transfer by zipping

 1def get_file_names_from_dir(directory, ext=".jpg"):
 2    file_list = []
 3    for root, dirnames, filenames in os.walk(directory):
 4        # Only load image files
 5        files = [f for f in filenames if f.lower().endswith(ext)]
 6        for filename in files:
 7            file_path = os.path.join(root, filename)
 8            if os.path.getsize(file_path) > 0:
 9                file_list.append(file_path)
10            else:
11                print(f"{filename} is zero length, so ignoring.")
12    return file_list

 1def make_dir_and_zip(num):
 2    root_dir = r"/tmp/dogs-vs-cats/split"
 3    dest_dir = f"{root_dir}/images_{num}"
 4    if os.path.exists(dest_dir):
 5        shutil.rmtree(dest_dir)
 6    if not os.path.exists(dest_dir):
 7        os.makedirs(dest_dir)
 8
 9    for a in animals_df.sample(num, random_state=42)["animals"]:
10        shutil.copy(a, dest_dir)
11
12    shutil.make_archive(dest_dir, "zip", dest_dir)

1animals = get_file_names_from_dir(r"/tmp/dogs-vs-cats")
2animals_df = pd.DataFrame({'animals': animals})
3
4for i in range(10, 100, 10):
5    print("- ", end="")
6    make_dir_and_zip(i)
7    make_dir_and_zip(i * 10)
8    make_dir_and_zip(i * 100)

1def copy_directory_and_files(source_path, dest_path):
2    print('Copying files')
3    start = time.time()
4    # Copy the content of source to destination
5    dest = shutil.copytree(source_path, dest_path)
6    copy_dir_time = (time.time() - start) * 1000
7    print(f'directory copied in {copy_dir_time:.0F} milliseconds')
8    num_copied = len(os.listdir(dest_path))
9    return copy_dir_time, num_copied

 1num = 10
 2source_dir = f"{image_remote_dir}images_{num}"
 3dest_dir = f"{local_image_dir}images_{num}"
 4copy_dir_time, num_copied = copy_directory_and_files(source_dir, dest_dir)
 5
 6"""
 7Copying files
 8directory copied in 4021 milliseconds
 9"""
10
11copy_dir_time
12"""
134021
14"""
15
16num_copied
17"""
1810
19"""

 1def copy_and_extract_data(zip_filename, source_path, dest_path, extract_path):
 2    zource_zip = source_path + zip_filename
 3    local_zip = dest_path + zip_filename
 4
 5    start = time.time()
 6    # Copy the content of source to destination
 7    dest = shutil.copyfile(zource_zip, local_zip)
 8    copy_time = (time.time() - start) * 1000
 9    print(f"Zip file copied in {copy_time:.0F} milliseconds")
10
11    start = time.time()
12    # Unzip the file
13    zip_ref = zipfile.ZipFile(local_zip, "r")
14    zip_ref.extractall(extract_path)
15    zip_ref.close()
16    unzip_time = (time.time() - start) * 1000
17    print(f"Zip file Extracted in {unzip_time:.0F} milliseconds")
18
19    num_extracted = len(os.listdir(extract_path))
20
21    return copy_time, unzip_time, num_extracted

 1num = 10
 2zip_filename = f"images_{num}.zip"
 3extract_path = f"{local_image_dir_zips}images_{num}"
 4copy_time, unzip_time, num_extracted = copy_and_extract_data(
 5    zip_filename, image_remote, local_image_dir_zips, extract_path
 6)
 7
 8"""
 9Zip file copied in 740 milliseconds
10Zip file Extracted in 5 milliseconds
11"""
12
13copy_time, unzip_time, num_extracted
14"""
15740, 5, 10
16"""

 1def copy_twice(num):
 2    print()
 3    print("*" * 30)
 4    print(f"num = {num}")
 5    zip_filename = f"images_{num}.zip"
 6    extract_path = f"{local_image_dir_zips}images_{num}"
 7    copy_time, unzip_time, num_extracted = copy_and_extract_data(
 8        zip_filename, image_remote, local_image_dir_zips, extract_path
 9    )
10    print()
11
12    source_dir = f"{image_remote}images_{num}"
13    dest_dir = f"{local_image_dir}images_{num}"
14    copy_dir_time, num_copied = copy_directory_and_files(source_dir, dest_dir)
15
16    r_df = pd.DataFrame(
17        [[num, copy_time, unzip_time, num_extracted, copy_dir_time, num_copied]],
18        columns=[
19            "num",
20            "copy_zip_time",
21            "unzip_time",
22            "num_extracted",
23            "copy_files_time",
24            "num_copied",
25        ],
26    )
27    return r_df

 1# Create the results dataframe
 2result_df = pd.DataFrame(
 3    [[0, 0, 0, 0, 0, 0]],
 4    columns=[
 5        "num",
 6        "copy_zip_time",
 7        "unzip_time",
 8        "num_extracted",
 9        "copy_files_time",
10        "num_copied",
11    ],
12)
13
14for num in range(10, 101, 10):
15    result_df = result_df.append(copy_twice(num))
16    result_df.to_csv(f"/content/gdrive/MyDrive/image_tests/timing_results_10_{num}.csv")
17
18for i in range(20, 101, 10):
19    num = i * 10
20    result_df = result_df.append(copy_twice(num))
21    result_df.to_csv(f"/content/gdrive/MyDrive/image_tests/timing_results_10_{num}.csv")
22
23for i in range(20, 51, 10):
24    num = i * 100
25    result_df = result_df.append(copy_twice(num))
26    result_df.to_csv(f"/content/gdrive/MyDrive/image_tests/timing_results_10_{num}.csv")

 1def plot_copy_time_vs_num_chart(df, title):
 2    fig, ax = plt.subplots(figsize=(8, 5), facecolor=plt.cm.Blues(0.2))
 3    fig.suptitle(title, fontsize="x-large", fontweight="bold")
 4    ax.set_facecolor(plt.cm.Blues(0.2))
 5    ax.plot(df.index, df.copy_files_time / 1000, label="Copy files")
 6    ax.plot(df.index, df.copy_unzip_time / 1000, label="Copy zip file and unzip")
 7    ax.legend(facecolor=plt.cm.Blues(0.1))
 8    ax.set_xlabel("Number of Files", fontsize="large")
 9    ax.set_ylabel("Time to transfer (seconds)", fontsize=14)
10    ax.spines["right"].set_visible(False)
11    ax.spines["top"].set_visible(False)
12    return fig
13
14
15fig = plot_copy_time_vs_num_chart(
16    res_df.loc[range(10, 101, 10), :],
17    "Time taken to transfer files vs number of files\n(10 to 100 files)",
18)
19
20fig = plot_copy_time_vs_num_chart(
21    res_df,
22    "Time taken to transfer files vs number of files\n(10 to 5000 files)",
23)