Update app.py
Browse files
app.py
CHANGED
|
@@ -3,19 +3,32 @@ import os
|
|
| 3 |
import json
|
| 4 |
import shutil
|
| 5 |
import gradio as gr
|
|
|
|
| 6 |
|
| 7 |
def process_and_zip_folders(huggingface_dataset_url, output_dir):
|
| 8 |
# URLからリポジトリIDを抽出
|
| 9 |
if huggingface_dataset_url.startswith("https://huggingface.co/"):
|
| 10 |
repo_id = huggingface_dataset_url.replace("https://huggingface.co/", "").rstrip("/")
|
| 11 |
-
# 'datasets/'がURLに含まれている場合、取り除く
|
| 12 |
if repo_id.startswith("datasets/"):
|
| 13 |
repo_id = repo_id[len("datasets/"):]
|
| 14 |
else:
|
| 15 |
repo_id = huggingface_dataset_url
|
| 16 |
|
| 17 |
-
# データセットをダウンロード (
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
# 処理対象のフォルダ
|
| 21 |
folders = [
|
|
|
|
| 3 |
import json
|
| 4 |
import shutil
|
| 5 |
import gradio as gr
|
| 6 |
+
import time
|
| 7 |
|
| 8 |
def process_and_zip_folders(huggingface_dataset_url, output_dir):
|
| 9 |
# URLからリポジトリIDを抽出
|
| 10 |
if huggingface_dataset_url.startswith("https://huggingface.co/"):
|
| 11 |
repo_id = huggingface_dataset_url.replace("https://huggingface.co/", "").rstrip("/")
|
|
|
|
| 12 |
if repo_id.startswith("datasets/"):
|
| 13 |
repo_id = repo_id[len("datasets/"):]
|
| 14 |
else:
|
| 15 |
repo_id = huggingface_dataset_url
|
| 16 |
|
| 17 |
+
# データセットをダウンロード (再試行ロジックを追加)
|
| 18 |
+
retry_attempts = 3
|
| 19 |
+
dataset_path = None
|
| 20 |
+
for attempt in range(retry_attempts):
|
| 21 |
+
try:
|
| 22 |
+
dataset_path = snapshot_download(repo_id, repo_type="dataset", force_download=True)
|
| 23 |
+
break # 成功した場合はループを終了
|
| 24 |
+
except OSError as e:
|
| 25 |
+
print(f"Download failed on attempt {attempt + 1}/{retry_attempts}: {e}")
|
| 26 |
+
if attempt == retry_attempts - 1:
|
| 27 |
+
raise # 最後の試行でも失敗した場合はエラーを再送出
|
| 28 |
+
time.sleep(5) # 少し待機して再試行
|
| 29 |
+
|
| 30 |
+
if dataset_path is None:
|
| 31 |
+
raise EnvironmentError("Failed to download dataset after multiple attempts.")
|
| 32 |
|
| 33 |
# 処理対象のフォルダ
|
| 34 |
folders = [
|