Sync with the main repo

apps/common/auto_zip.py

@@ -0,0 +1,53 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import json
+import os
+import zipfile
+
+
+class AutoZip:
+
+    def __init__(self, zip_path: str, ext: str = ".json"):
+        self.zip_path = zip_path
+        self.zip = zipfile.ZipFile(zip_path, "r")
+        self.fl = [f for f in self.zip.filelist if f.filename.endswith(ext)]
+
+    def __next__(self):
+        if self.index >= len(self.fl):
+            raise StopIteration
+        else:
+            finfo = self.fl[self.index]
+            with self.zip.open(finfo) as f:
+                raw_json = json.loads(f.read().decode("utf-8"))
+            self.index += 1
+            return raw_json
+
+    def __len__(self):
+        return len(self.fl)
+
+    def __iter__(self):
+        self.index = 0
+        return self
+
+    def as_dict(self, include_zip_name: bool = False):
+        d = dict()
+        for finfo in self.fl:
+            with self.zip.open(finfo) as f:
+                raw_text = f.read().decode("utf-8")
+            if include_zip_name:
+                key = os.path.split(self.zip_path)[1] + "/" + finfo.filename
+            else:
+                key = finfo.filename
+            d[key] = raw_text
+        return d

apps/data_explorer/data_explorer.py

@@ -1,3 +1,16 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 """
 Gradio-based web UI to explore the Camel dataset.
 """
@@ -39,7 +52,8 @@ def parse_arguments():
     return args
 
 
-def construct_ui(blocks, datasets: Datasets, default_dataset: str = None):
+def construct_ui(blocks, datasets: Datasets,
+                 default_dataset: Optional[str] = None):
     """ Build Gradio UI and populate with chat data from JSONs.
 
     Args:
@@ -213,7 +227,7 @@ def construct_ui(blocks, datasets: Datasets, default_dataset: str = None):
         Returns:
             List[Tuple]: Chat history in chatbot UI element format.
         """
-        history = []
+        history: List[Tuple] = []
         curr_qa = (None, None)
         for k in sorted(messages.keys()):
             msg = messages[k]

apps/data_explorer/downloader.py

@@ -1,7 +1,21 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 import os
 import urllib.request
 
 from huggingface_hub import hf_hub_download
+from huggingface_hub.utils._errors import RepositoryNotFoundError
 
 REPO_ROOT = os.path.realpath(
     os.path.join(os.path.dirname(os.path.abspath(__file__)), "../.."))
@@ -23,7 +37,7 @@ def download_data():
         hf_hub_download(repo_id="camel-ai/code", repo_type="dataset",
                         filename="code_chat.zip", local_dir=data_dir,
                         local_dir_use_symlinks=False)
-    except:
+    except RepositoryNotFoundError:
         for name in ("ai_society_chat.zip", "code_chat.zip"):
             data_url = ("https://storage.googleapis.com/"
                         f"camel-bucket/datasets/private/{name}")

apps/data_explorer/loader.py

@@ -1,16 +1,29 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 """
 Everything related to parsing the data JSONs into UI-compatible format.
 """
 
 import glob
-import json
 import os
 import re
-import zipfile
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, Optional, Tuple, Union
 
 from tqdm import tqdm
 
+from apps.common.auto_zip import AutoZip
+
 ChatHistory = Dict[str, Any]
 ParsedChatHistory = Dict[str, Any]
 AllChats = Dict[str, Any]
@@ -20,30 +33,6 @@ REPO_ROOT = os.path.realpath(
     os.path.join(os.path.dirname(os.path.abspath(__file__)), "../.."))
 
 
-class AutoZip:
-    def __init__(self, zip_path: str, ext: str = ".json"):
-        self.zip_path = zip_path
-        self.zip = zipfile.ZipFile(zip_path, "r")
-        self.fl = [f for f in self.zip.filelist if f.filename.endswith(ext)]
-
-    def __next__(self):
-        if self.index >= len(self.fl):
-            raise StopIteration
-        else:
-            finfo = self.fl[self.index]
-            with self.zip.open(finfo) as f:
-                raw_json = json.loads(f.read().decode("utf-8"))
-            self.index += 1
-            return raw_json
-
-    def __len__(self):
-        return len(self.fl)
-
-    def __iter__(self):
-        self.index = 0
-        return self
-
-
 def parse(raw_chat: ChatHistory) -> Union[ParsedChatHistory, None]:
     """ Gets the JSON raw chat data, validates it and transforms
         into an easy to work with form.
@@ -122,17 +111,17 @@ def load_zip(zip_path: str) -> AllChats:
             continue
         parsed_list.append(parsed)
 
-    assistant_roles = set()
-    user_roles = set()
+    assistant_roles_set = set()
+    user_roles_set = set()
     for parsed in parsed_list:
-        assistant_roles.add(parsed['assistant_role'])
-        user_roles.add(parsed['user_role'])
-    assistant_roles = list(sorted(assistant_roles))
-    user_roles = list(sorted(user_roles))
-    matrix: Dict[Tuple[str, str], List[Dict]] = dict()
+        assistant_roles_set.add(parsed['assistant_role'])
+        user_roles_set.add(parsed['user_role'])
+    assistant_roles = list(sorted(assistant_roles_set))
+    user_roles = list(sorted(user_roles_set))
+    matrix: Dict[Tuple[str, str], Dict[str, Dict]] = dict()
     for parsed in parsed_list:
         key = (parsed['assistant_role'], parsed['user_role'])
-        original_task = parsed['original_task']
+        original_task: str = parsed['original_task']
         new_item = {
             k: v
             for k, v in parsed.items()

sync.sh

@@ -0,0 +1,15 @@
+TMP_DIR=/tmp/camel_hf_tmp
+echo $TMP_DIR
+HF_REPO_DIR=`realpath .`
+echo $HF_REPO_DIR
+
+mkdir -p $TMP_DIR
+git clone -b hf_spaces_2 https://github.com/lightaime/camel.git $TMP_DIR
+cd $TMP_DIR
+
+find apps/data_explorer -name "*.py" | grep -v test | xargs -n 1 -I {} rsync -R {} $HF_REPO_DIR
+find apps/common -name "*.py" | grep -v test | xargs -n 1 -I {} rsync -R {} $HF_REPO_DIR
+
+rm -rf $TMP_DIR
+
+echo Done