diff --git a/parts/0_Background/blue.png b/parts/01_Background/blue.png similarity index 100% rename from parts/0_Background/blue.png rename to parts/01_Background/blue.png diff --git a/parts/0_Background/green.png b/parts/01_Background/green.png similarity index 100% rename from parts/0_Background/green.png rename to parts/01_Background/green.png diff --git a/parts/0_Background/purple.png b/parts/01_Background/purple.png similarity index 100% rename from parts/0_Background/purple.png rename to parts/01_Background/purple.png diff --git a/parts/0_Background/red.png b/parts/01_Background/red.png similarity index 100% rename from parts/0_Background/red.png rename to parts/01_Background/red.png diff --git a/parts/0_Background/white.png b/parts/01_Background/white.png similarity index 100% rename from parts/0_Background/white.png rename to parts/01_Background/white.png diff --git a/parts/2_First Latter/B.png b/parts/02_First Letter/B.png similarity index 100% rename from parts/2_First Latter/B.png rename to parts/02_First Letter/B.png diff --git a/parts/2_First Latter/C.png b/parts/02_First Letter/C.png similarity index 100% rename from parts/2_First Latter/C.png rename to parts/02_First Letter/C.png diff --git a/parts/2_First Latter/D.png b/parts/02_First Letter/D.png similarity index 100% rename from parts/2_First Latter/D.png rename to parts/02_First Letter/D.png diff --git a/parts/2_First Latter/E.png b/parts/02_First Letter/E.png similarity index 100% rename from parts/2_First Latter/E.png rename to parts/02_First Letter/E.png diff --git a/parts/2_First Latter/F.png b/parts/02_First Letter/F.png similarity index 100% rename from parts/2_First Latter/F.png rename to parts/02_First Letter/F.png diff --git a/parts/2_First Latter/G.png b/parts/02_First Letter/G.png similarity index 100% rename from parts/2_First Latter/G.png rename to parts/02_First Letter/G.png diff --git a/parts/2_First Latter/H.png b/parts/02_First Letter/H.png similarity index 100% rename from parts/2_First Latter/H.png rename to parts/02_First Letter/H.png diff --git a/parts/2_First Latter/I.png b/parts/02_First Letter/I.png similarity index 100% rename from parts/2_First Latter/I.png rename to parts/02_First Letter/I.png diff --git a/parts/2_First Latter/J.png b/parts/02_First Letter/J.png similarity index 100% rename from parts/2_First Latter/J.png rename to parts/02_First Letter/J.png diff --git a/parts/2_First Latter/K.png b/parts/02_First Letter/K.png similarity index 100% rename from parts/2_First Latter/K.png rename to parts/02_First Letter/K.png diff --git a/parts/1_Second Letter/A.png b/parts/03_Second Letter/A.png similarity index 100% rename from parts/1_Second Letter/A.png rename to parts/03_Second Letter/A.png diff --git a/parts/1_Second Letter/Q.png b/parts/03_Second Letter/Q.png similarity index 100% rename from parts/1_Second Letter/Q.png rename to parts/03_Second Letter/Q.png diff --git a/parts/1_Second Letter/R.png b/parts/03_Second Letter/R.png similarity index 100% rename from parts/1_Second Letter/R.png rename to parts/03_Second Letter/R.png diff --git a/parts/1_Second Letter/T.png b/parts/03_Second Letter/T.png similarity index 100% rename from parts/1_Second Letter/T.png rename to parts/03_Second Letter/T.png diff --git a/parts/1_Second Letter/V.png b/parts/03_Second Letter/V.png similarity index 100% rename from parts/1_Second Letter/V.png rename to parts/03_Second Letter/V.png diff --git a/parts/1_Second Letter/W.png b/parts/03_Second Letter/W.png similarity index 100% rename from parts/1_Second Letter/W.png rename to parts/03_Second Letter/W.png diff --git a/parts/1_Second Letter/X.png b/parts/03_Second Letter/X.png similarity index 100% rename from parts/1_Second Letter/X.png rename to parts/03_Second Letter/X.png diff --git a/parts/1_Second Letter/Y.png b/parts/03_Second Letter/Y.png similarity index 100% rename from parts/1_Second Letter/Y.png rename to parts/03_Second Letter/Y.png diff --git a/parts/1_Second Letter/Z.png b/parts/03_Second Letter/Z.png similarity index 100% rename from parts/1_Second Letter/Z.png rename to parts/03_Second Letter/Z.png diff --git a/parts/1_Second Letter/p.png b/parts/03_Second Letter/p.png similarity index 100% rename from parts/1_Second Letter/p.png rename to parts/03_Second Letter/p.png diff --git a/parts2/0_Background/black.png b/parts2/01_Background/black.png similarity index 100% rename from parts2/0_Background/black.png rename to parts2/01_Background/black.png diff --git a/parts2/2_First Latter/1.png b/parts2/02_First Letter/1.png similarity index 100% rename from parts2/2_First Latter/1.png rename to parts2/02_First Letter/1.png diff --git a/parts2/2_First Latter/2.png b/parts2/02_First Letter/2.png similarity index 100% rename from parts2/2_First Latter/2.png rename to parts2/02_First Letter/2.png diff --git a/parts2/2_First Latter/3.png b/parts2/02_First Letter/3.png similarity index 100% rename from parts2/2_First Latter/3.png rename to parts2/02_First Letter/3.png diff --git a/parts2/1_Second Letter/empty.png b/parts2/02_First Letter/empty.png similarity index 100% rename from parts2/1_Second Letter/empty.png rename to parts2/02_First Letter/empty.png diff --git a/parts2/2_First Latter/empty.png b/parts2/03_Second Letter/empty.png similarity index 100% rename from parts2/2_First Latter/empty.png rename to parts2/03_Second Letter/empty.png diff --git "a/parts2/1_Second Letter/\302\245.png" "b/parts2/03_Second Letter/\302\245.png" similarity index 100% rename from "parts2/1_Second Letter/\302\245.png" rename to "parts2/03_Second Letter/\302\245.png" diff --git a/ratio.csv b/ratio.csv index 3537976..7942b18 100644 --- a/ratio.csv +++ b/ratio.csv @@ -4,6 +4,16 @@ parts,Background,green,1 parts,Background,purple,1 parts,Background,red,1 parts,Background,white,1 +parts,First Letter,B,1 +parts,First Letter,C,1 +parts,First Letter,D,1 +parts,First Letter,E,1 +parts,First Letter,F,1 +parts,First Letter,G,1 +parts,First Letter,H,1 +parts,First Letter,I,1 +parts,First Letter,J,1 +parts,First Letter,K,1 parts,Second Letter,A,1 parts,Second Letter,Q,1 parts,Second Letter,R,1 @@ -14,20 +24,10 @@ parts,Second Letter,X,1 parts,Second Letter,Y,1 parts,Second Letter,Z,1 parts,Second Letter,p,1 -parts,First Latter,B,1 -parts,First Latter,C,1 -parts,First Latter,D,1 -parts,First Latter,E,1 -parts,First Latter,F,1 -parts,First Latter,G,1 -parts,First Latter,H,1 -parts,First Latter,I,1 -parts,First Latter,J,1 -parts,First Latter,K,1 parts2,Background,black,1 +parts2,First Letter,1,1 +parts2,First Letter,2,1 +parts2,First Letter,3,1 +parts2,First Letter,empty,1 parts2,Second Letter,empty,1 parts2,Second Letter,¥,1 -parts2,First Latter,1,1 -parts2,First Latter,2,1 -parts2,First Latter,3,1 -parts2,First Latter,empty,1 diff --git a/src/config.py b/src/config.py index 2f7be9c..7ca1932 100644 --- a/src/config.py +++ b/src/config.py @@ -60,7 +60,6 @@ class Quality(Enum): # upload---------------------------------------------------------------------------------------------- -READ_IMAGES = False # if read images from image_ipfs_data.backup, set True UPLOAD_METADATA = False # set False if don't want to upload metadata PIN_FILES = False # if want to upload permanently, set to True PROXIES = { diff --git a/src/fresh_metadata.py b/src/fresh_metadata.py index 0fa0fce..339509d 100644 --- a/src/fresh_metadata.py +++ b/src/fresh_metadata.py @@ -1,4 +1,3 @@ -from typing import List from httpx import Timeout, Client, AsyncClient import time import asyncio @@ -145,7 +144,7 @@ def opensea_refresh(client: Client, id: int) -> bool: return False -async def ipfs_query_tasks(request_urls: List[str]): +async def ipfs_query_tasks(request_urls: list[str]): """ create asyncio tasks for ipfs query diff --git a/src/generate.py b/src/generate.py index 3c23c21..7926beb 100644 --- a/src/generate.py +++ b/src/generate.py @@ -1,5 +1,4 @@ import os -from typing import List from PIL import Image import pandas as pd import numpy as np @@ -194,7 +193,7 @@ def generate_images( return df_attr -def check_values_valid(df: pd.DataFrame, select_columns: List, all_values: List): +def check_values_valid(df: pd.DataFrame, select_columns: list, all_values: list): """ Check selected column values is in the list diff --git a/src/get_table.py b/src/get_table.py index 0d19a64..0d8e889 100644 --- a/src/get_table.py +++ b/src/get_table.py @@ -3,6 +3,7 @@ import pandas as pd import shutil from config import FOLDERS, EXTENSION, W, H, WEIGHTS +from math import fsum def get_files_path(folders=FOLDERS): @@ -46,7 +47,7 @@ def get_files_path(folders=FOLDERS): # Validate weights assert ( - sum(WEIGHTS) == 1 + fsum(WEIGHTS) == 1 ), f"sum of PARTS_DICT's value in config.py should be 1, now is {sum(WEIGHTS)}" # Validate image format and size @@ -70,7 +71,7 @@ def get_files_path(folders=FOLDERS): # Validate path name has - for path in files_path: assert ( - "-" not in path.split("/")[-1] + "-" not in path.split(os.sep)[-1] ), f"{path} is invalid, files should not have '-' symbol" folder_set = set() diff --git a/src/upload.py b/src/upload.py index 1ccf614..ae7a088 100644 --- a/src/upload.py +++ b/src/upload.py @@ -1,6 +1,9 @@ +import asyncio import os -import httpx +from httpx import AsyncClient, Limits, ReadTimeout, Client import json +import pandas as pd +import random from config import ( IMAGES, METADATA, @@ -9,13 +12,65 @@ PROJECT_ID, PROJECT_SECRET, PROXIES, - READ_IMAGES, UPLOAD_METADATA, PIN_FILES, ) -import pandas as pd from final_check import RENAME_DF, START_ID -import random + + +async def upload_task(files_path_chunk: list[str], wait_seconds: int) -> list[dict]: + """ + upload task for asyncio, a task process 10 files + + Args: + files_path_chunk (list[str]): a list contain 10 files path + wait_seconds (int): because infura limit, 1 second can post 10 times, add wait_seconds to wait + + Returns: + list[dict]: 10 files ipfs info + """ + await asyncio.sleep(wait_seconds) + async with AsyncClient( + proxies=PROXIES, limits=Limits(max_connections=10), timeout=60 + ) as client: + loop = asyncio.get_running_loop() + tasks = [ + loop.create_task(upload_single_async(client, file_path)) + for file_path in files_path_chunk + ] + result = await asyncio.gather(*tasks) + return result + + +async def upload_single_async(client: AsyncClient, file_path: str) -> dict: + """ + upload folder to ipfs + + Args: + client (AsyncClient): httpx.asyncClient instance + file_path (str): path of file want to upload + + Returns: + dict: ipfs info json + """ + retry = 0 + while retry < 5: + try: + response = await client.post( + f"https://ipfs.infura.io:5001/api/v0/add", + params={ + "pin": "true" if PIN_FILES else "false" + }, # pin=true if want to pin files + auth=(PROJECT_ID, PROJECT_SECRET), + files={"file": open(file_path, "rb")}, + ) + res_json = response.json() + if res_json["Name"] != "": + return res_json + except Exception as e: + if isinstance(e, ReadTimeout): + print(f"upload {file_path.split('-')[0]} timeout, retry {retry}") + retry += 1 def upload_folder( @@ -32,39 +87,24 @@ def upload_folder( tuple[str, list[dict]]: (folder_hash, images_dict_list) """ files = [] - if content_type == "image/png": - files = [ - ( - folder_name.split(os.sep)[-1], - (file, open(os.path.join(folder_name, file), "rb"), content_type), - ) - for file in list( - filter(lambda i: i.split(".")[-1] == "png", os.listdir(folder_name)) - ) - ] - elif content_type == "image/gif": - files = [ - ( - folder_name.split(os.sep)[-1], - (file, open(os.path.join(folder_name, file), "rb"), content_type), - ) - for file in list( - filter(lambda i: i.split(".")[-1] == "gif", os.listdir(folder_name)) - ) - ] - elif content_type == "application/json": - files = [ - ( - folder_name.split(os.sep)[-1], - (file, open(os.path.join(folder_name, file), "rb"), content_type), - ) - for file in list(filter(lambda i: "." not in i, os.listdir(folder_name))) - ] + extension = content_type.split(os.sep)[-1] - with httpx.Client(proxies=PROXIES) as client: + files = [ + (file, open(os.path.join(folder_name, file), "rb")) + for file in list( + filter(lambda i: i.split(".")[-1] == extension, os.listdir(folder_name)) + ) + ] + + with Client(proxies=PROXIES, timeout=None) as client: response = client.post( - f"https://ipfs.infura.io:5001/api/v0/add?pin={'true' if PIN_FILES else 'false'}&recursive=true&wrap-with-directory=true", # pin=true if want to pin files - files=files, + f"https://ipfs.infura.io:5001/api/v0/add", + params={ + "pin": "true" if PIN_FILES else "false", + "recursive": "true", + "wrap-with-directory": "true", + }, + files=files, # files should be List[filename, bytes] auth=(PROJECT_ID, PROJECT_SECRET), ) upload_folder_res_list = response.text.strip().split("\n") @@ -85,9 +125,49 @@ def upload_folder( return (folder_hash, images_dict_list) +def upload_files(folder_name: str, content_type: str = "image/png") -> list[dict]: + """ + upload files in a folder to ipfs + + Args: + folder_name (str): files in folder + content_type (str, optional): mime file type. Defaults to "image/png". + + Returns: + list[dict]: ipfs info list, example: [{ 'Name': str, 'Hash': str, 'Size': str }] + """ + extension = content_type.split(os.sep)[-1] + file_paths = [ + os.path.join(folder_name, file_path) + for file_path in list( + filter(lambda i: i.split(".")[-1] == extension, os.listdir(folder_name)) + ) + ] + file_count = len(file_paths) + chunk_size = 10 # 10 per second for infura + chunks = [file_paths[i : i + chunk_size] for i in range(0, file_count, chunk_size)] + tasks = [] + results = [] + + def complete_batch_callback(images_ipfs_data): + results.append(images_ipfs_data.result()) + print(f"complete {len(results)/len(chunks):.2%}") + + loop = asyncio.get_event_loop() + print(f"Total {len(file_count)} files to upload, estimate time: {len(chunks)+10}s") + for epoch, path_chunk in enumerate(chunks): + task = loop.create_task(upload_task(path_chunk, epoch)) + tasks.append(task) + task.add_done_callback(complete_batch_callback) + + loop.run_until_complete(asyncio.wait(tasks)) + print(f"upload {len(results)} files complete.") + return results + + def generate_metadata( df: pd.DataFrame, - image_ipfs_data: dict, + image_ipfs_data: list[dict], start_id: int = 0, image_folder: str = IMAGES, metadata_folder: str = METADATA, @@ -133,26 +213,58 @@ def generate_metadata( "attributes": attributes, } info_json = json.dumps(info_dict) - with open(os.path.join(metadata_folder, str(index)), "w") as f: + with open(os.path.join(metadata_folder, str(index) + ".json"), "w") as f: f.write(info_json) return (start_id, start_id + len(df) - 1) +def read_images_from_local() -> list[dict]: + """ + read images from local pickle + + Returns: + list[dict]: images ipfs info + """ + with open("image_ipfs_data.backup", "r") as f: + result = json.loads(f.read()) + print(f"read {len(result)} ipfs data from local") + return result + + +def download_and_save(): + """ + upload images and get ipfs info + + Returns: + list[dict]: images ipfs info + """ + all_ipfs_info_batch = upload_files(IMAGES) + image_ipfs_data = [] + for batch_info in all_ipfs_info_batch: + for single_info in batch_info: + image_ipfs_data.append(single_info) + with open("image_ipfs_data.backup", "w") as f: + f.write(json.dumps(image_ipfs_data)) + print("save image_ipfs_data to image_ipfs_data.backup") + return image_ipfs_data + + if __name__ == "__main__": - df = RENAME_DF - if not READ_IMAGES: - image_ipfs_root, image_ipfs_data = upload_folder(IMAGES) - print(f"image_ipfs_root: {image_ipfs_root}") - # backup file use for debug upload images data - with open("image_ipfs_data.backup", "w") as f: - f.write(json.dumps(image_ipfs_data)) - print("save image_ipfs_data to image_ipfs_data.backup") + if not PIN_FILES: + print( + f"Pin file is {PIN_FILES}, set PIN_FILES=True in config.py if want to pin files" + ) + if os.path.exists("image_ipfs_data.backup"): + use_local = input("image_ipfs_data.backup exist, load from local? (y/n)") + if use_local == "y": + image_ipfs_data = read_images_from_local() + else: + image_ipfs_data = download_and_save() else: - # if read images hashes from backup - with open("image_ipfs_data.backup", "r") as j: - image_ipfs_data = json.loads(j.read()) + image_ipfs_data = download_and_save() + df = RENAME_DF start, end = generate_metadata(df, image_ipfs_data, START_ID) print(f"Generate metadata complete, Index from {start} to {end}") @@ -161,5 +273,5 @@ def generate_metadata( metadata_root, _ = upload_folder(METADATA, "application/json") print(f"upload metadatas complete") print( - f"Source url is {metadata_root}, you can visit ipfs://{metadata_root}/{start} to check" + f"Source url is {metadata_root}, you can visit ipfs://{metadata_root}/{start}.json to check" )