Merge pull request #26 from laelhalawani/develop

Develop
laelhalawani · Jan 5, 2024 · d63bfb7 · d63bfb7
2 parents 93613ac + 5a53951
commit d63bfb7
Show file tree

Hide file tree

Showing 13 changed files with 170 additions and 245 deletions.
diff --git a/docs/glai/ai/auto_ai.html b/docs/glai/ai/auto_ai.html
@@ -61,8 +61,7 @@ <h1 class="title">Module <code>glai.ai.auto_ai</code></h1>
                  name_search: Optional[str] = None,
                  quantization_search: Optional[str] = None,
                  keyword_search: Optional[str] = None,
-                 new_tokens: int = 1500,
-                 max_input_tokens: int = 900,
+                 max_total_tokens: int = 1500,
                  model_db_dir:str = DEFAULT_LOCAL_GGUF_DIR) -&gt; None:
 
         self.ai_db = ModelDB(model_db_dir=model_db_dir, copy_examples=True)
@@ -71,7 +70,7 @@ <h1 class="title">Module <code>glai.ai.auto_ai</code></h1>
         )
         self.model_data.download_gguf()
         self.ai = LlamaAI(
-            self.model_data.gguf_file_path, new_tokens, max_input_tokens
+            self.model_data.gguf_file_path, max_tokens=max_total_tokens
         )
         print(f&#34;Using model: {self.model_data}&#34;)
         self.msgs: AIMessages = AIMessages(
@@ -202,7 +201,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 <dl>
 <dt id="glai.ai.auto_ai.AutoAI"><code class="flex name class">
 <span>class <span class="ident">AutoAI</span></span>
-<span>(</span><span>name_search: Optional[str] = None, quantization_search: Optional[str] = None, keyword_search: Optional[str] = None, new_tokens: int = 1500, max_input_tokens: int = 900, model_db_dir: str = './gguf_db')</span>
+<span>(</span><span>name_search: Optional[str] = None, quantization_search: Optional[str] = None, keyword_search: Optional[str] = None, max_total_tokens: int = 1500, model_db_dir: str = './gguf_db')</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Initialize the AutoAI class for super easy LLM AI generation.</p>
@@ -264,8 +263,7 @@ <h2 id="attributes">Attributes</h2>
                  name_search: Optional[str] = None,
                  quantization_search: Optional[str] = None,
                  keyword_search: Optional[str] = None,
-                 new_tokens: int = 1500,
-                 max_input_tokens: int = 900,
+                 max_total_tokens: int = 1500,
                  model_db_dir:str = DEFAULT_LOCAL_GGUF_DIR) -&gt; None:
 
         self.ai_db = ModelDB(model_db_dir=model_db_dir, copy_examples=True)
@@ -274,7 +272,7 @@ <h2 id="attributes">Attributes</h2>
         )
         self.model_data.download_gguf()
         self.ai = LlamaAI(
-            self.model_data.gguf_file_path, new_tokens, max_input_tokens
+            self.model_data.gguf_file_path, max_tokens=max_total_tokens
         )
         print(f&#34;Using model: {self.model_data}&#34;)
         self.msgs: AIMessages = AIMessages(

diff --git a/docs/glai/ai/easy_ai.html b/docs/glai/ai/easy_ai.html
diff --git a/docs/glai/ai/index.html b/docs/glai/ai/index.html
@@ -57,7 +57,7 @@ <h2 class="section-title" id="header-classes">Classes</h2>
 <dl>
 <dt id="glai.ai.AutoAI"><code class="flex name class">
 <span>class <span class="ident">AutoAI</span></span>
-<span>(</span><span>name_search: Optional[str] = None, quantization_search: Optional[str] = None, keyword_search: Optional[str] = None, new_tokens: int = 1500, max_input_tokens: int = 900, model_db_dir: str = './gguf_db')</span>
+<span>(</span><span>name_search: Optional[str] = None, quantization_search: Optional[str] = None, keyword_search: Optional[str] = None, max_total_tokens: int = 1500, model_db_dir: str = './gguf_db')</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Initialize the AutoAI class for super easy LLM AI generation.</p>
@@ -119,8 +119,7 @@ <h2 id="attributes">Attributes</h2>
                  name_search: Optional[str] = None,
                  quantization_search: Optional[str] = None,
                  keyword_search: Optional[str] = None,
-                 new_tokens: int = 1500,
-                 max_input_tokens: int = 900,
+                 max_total_tokens: int = 1500,
                  model_db_dir:str = DEFAULT_LOCAL_GGUF_DIR) -&gt; None:
 
         self.ai_db = ModelDB(model_db_dir=model_db_dir, copy_examples=True)
@@ -129,7 +128,7 @@ <h2 id="attributes">Attributes</h2>
         )
         self.model_data.download_gguf()
         self.ai = LlamaAI(
-            self.model_data.gguf_file_path, new_tokens, max_input_tokens
+            self.model_data.gguf_file_path, max_tokens=max_total_tokens
         )
         print(f&#34;Using model: {self.model_data}&#34;)
         self.msgs: AIMessages = AIMessages(
@@ -456,7 +455,7 @@ <h2 id="returns">Returns</h2>
 <p>Step by step call these with appropriate arguments:</p>
 <pre><code>1. `self.load_model_db(model_db_dir: str = DEFAULT_LOCAL_GGUF_DIR)`
 2. One of the following with necessary args: &lt;code&gt;self.model\_data\_from\_url()&lt;/code&gt; or &lt;code&gt;self.model\_data\_from\_file()&lt;/code&gt; or &lt;code&gt;self.find\_model\_data()&lt;/code&gt;
-3. `self.load_ai(max_tokens: int = 200, max_input_tokens: int = 100)`
+3. `self.load_ai(max_total_tokens: int = 200)`
 </code></pre>
 <p>All in one:</p>
 <pre><code>```python
@@ -467,8 +466,7 @@ <h2 id="returns">Returns</h2>
     name_search: Optional[str] = None, 
     quantization_search: Optional[str] = None, 
     keyword_search: Optional[str] = None, 
-    new_tokens: int = 200, 
-    max_input_tokens: int = 100
+    max_total_tokens: int = 200, 
     )
 ```
 
@@ -479,8 +477,7 @@ <h2 id="returns">Returns</h2>
 - quantization_search: Quantization of model to search for in the model db dir..(Optional)
 - keyword_search: Keyword of model to search for in the model db dir..(Optional)
 - model_gguf_path: Path to GGUF file of model to configure with.(Optional, not a recommended method, doesn't preserve download url)
-- new_tokens: Max tokens to be generated by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
-- max_input_tokens: Max input tokens to be generated by LlamaAI model. (Defaults to 100, set to around 200-300 for regular use)
+- max_total_tokens: Max tokens to be processed by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
 </code></pre>
 <h2 id="attributes">Attributes</h2>
 <dl>
@@ -521,7 +518,7 @@ <h2 id="methods">Methods</h2>
     Step by step call these with appropriate arguments:\n
         1. `self.load_model_db(model_db_dir: str = DEFAULT_LOCAL_GGUF_DIR)`
         2. One of the following with necessary args: `self.model_data_from_url()` or `self.model_data_from_file()` or `self.find_model_data()`
-        3. `self.load_ai(max_tokens: int = 200, max_input_tokens: int = 100)`
+        3. `self.load_ai(max_total_tokens: int = 200)`
     All in one:\n
         ```python
         self.configure(
@@ -531,8 +528,7 @@ <h2 id="methods">Methods</h2>
             name_search: Optional[str] = None, 
             quantization_search: Optional[str] = None, 
             keyword_search: Optional[str] = None, 
-            new_tokens: int = 200, 
-            max_input_tokens: int = 100
+            max_total_tokens: int = 200, 
             )
         ```
 
@@ -543,8 +539,7 @@ <h2 id="methods">Methods</h2>
         - quantization_search: Quantization of model to search for in the model db dir..(Optional)
         - keyword_search: Keyword of model to search for in the model db dir..(Optional)
         - model_gguf_path: Path to GGUF file of model to configure with.(Optional, not a recommended method, doesn&#39;t preserve download url)
-        - new_tokens: Max tokens to be generated by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
-        - max_input_tokens: Max input tokens to be generated by LlamaAI model. (Defaults to 100, set to around 200-300 for regular use)
+        - max_total_tokens: Max tokens to be processed by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
 
     Attributes:
         model_db: ModelDB for searching/loading models
@@ -583,8 +578,8 @@ <h2 id="methods">Methods</h2>
                   name_search: Optional[str] = None,
                   quantization_search: Optional[str] = None,
                   keyword_search: Optional[str] = None,
-                  new_tokens: int = 200,
-                  max_input_tokens: int = 100) -&gt; None:
+                  max_total_tokens: int = 200,
+                                            ) -&gt; None:
         &#34;&#34;&#34;
         Configure EasyAI with model data.
 
@@ -593,9 +588,8 @@ <h2 id="methods">Methods</h2>
 
         Args:
             model_db_dir: Directory to store model data in. Defaults to DEFAULT_LOCAL_GGUF_DIR.
-            max_tokens: Max tokens to be generated by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
-            max_input_tokens: Max input tokens to be generated by LlamaAI model. (Defaults to 100, set to around 200-300 for regular use)
-
+            max_total_tokens: Max tokens to be processed (input+generation) by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
+
             Provide at least one of these args to fetch ModelData: 
             ---
             model_url: URL of model to configure with. Automatically downloads and builds as needed. (Optional)
@@ -622,7 +616,7 @@ <h2 id="methods">Methods</h2>
         else:
             raise Exception(&#34;Can&#39;t find model data. Please provide a model URL, GGUF file path, or model name/quantization/keyword.&#34;)
 
-        self.load_ai(new_tokens, max_input_tokens)
+        self.load_ai(max_total_tokens)
 
 
 
@@ -743,17 +737,14 @@ <h2 id="methods">Methods</h2>
         self.messages = AIMessages(user_tags=self.model_data.user_tags, ai_tags=self.model_data.ai_tags)
 
     def load_ai(self,
-                max_tokens: int = 200,
-                max_input_tokens: int = 100) -&gt; None:
+                max_total_tokens: int = 200,) -&gt; None:
         &#34;&#34;&#34;
         Load LlamaAI model from model data.
 
         Downloads model file from model data URL if needed. Initializes LlamaAI with model and sets lai attribute.
 
         Args:
-            max_tokens: Max tokens for LlamaAI model.
-            max_input_tokens: Max input tokens for LlamaAI model.
-
+            max_total_tokens: Max tokens for LlamaAI model.
         Raises:
             Exception: If no model data or messages loaded yet.
         &#34;&#34;&#34;
@@ -763,7 +754,7 @@ <h2 id="methods">Methods</h2>
         if self.model_data is None:
             raise Exception(&#34;No model data loaded. Use find_model_data(), get_model_data_from_url(), or get_model_data_from_file() first.&#34;)
         self.model_data.download_gguf()
-        self.ai = LlamaAI(self.model_data.model_path(), max_tokens=max_tokens, max_input_tokens=max_input_tokens)
+        self.ai = LlamaAI(self.model_data.model_path(), max_tokens=max_total_tokens)
         print(f&#34;Loaded: {self.model_data}&#34;)
 
     def generate(self,
@@ -865,7 +856,7 @@ <h2 id="methods">Methods</h2>
 <h3>Methods</h3>
 <dl>
 <dt id="glai.ai.EasyAI.configure"><code class="name flex">
-<span>def <span class="ident">configure</span></span>(<span>self, model_db_dir: str = './gguf_db', model_url: Optional[str] = None, model_gguf_path: Optional[str] = None, name_search: Optional[str] = None, quantization_search: Optional[str] = None, keyword_search: Optional[str] = None, new_tokens: int = 200, max_input_tokens: int = 100) ‑> None</span>
+<span>def <span class="ident">configure</span></span>(<span>self, model_db_dir: str = './gguf_db', model_url: Optional[str] = None, model_gguf_path: Optional[str] = None, name_search: Optional[str] = None, quantization_search: Optional[str] = None, keyword_search: Optional[str] = None, max_total_tokens: int = 200) ‑> None</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Configure EasyAI with model data.</p>
@@ -875,10 +866,8 @@ <h2 id="args">Args</h2>
 <dl>
 <dt><strong><code>model_db_dir</code></strong></dt>
 <dd>Directory to store model data in. Defaults to DEFAULT_LOCAL_GGUF_DIR.</dd>
-<dt><strong><code>max_tokens</code></strong></dt>
-<dd>Max tokens to be generated by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)</dd>
-<dt><strong><code>max_input_tokens</code></strong></dt>
-<dd>Max input tokens to be generated by LlamaAI model. (Defaults to 100, set to around 200-300 for regular use)</dd>
+<dt><strong><code>max_total_tokens</code></strong></dt>
+<dd>Max tokens to be processed (input+generation) by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)</dd>
 </dl>
 <h2 id="provide-at-least-one-of-these-args-to-fetch-modeldata">Provide at least one of these args to fetch ModelData:</h2>
 <dl>
@@ -912,8 +901,8 @@ <h2 id="raises">Raises</h2>
               name_search: Optional[str] = None,
               quantization_search: Optional[str] = None,
               keyword_search: Optional[str] = None,
-              new_tokens: int = 200,
-              max_input_tokens: int = 100) -&gt; None:
+              max_total_tokens: int = 200,
+                                        ) -&gt; None:
     &#34;&#34;&#34;
     Configure EasyAI with model data.
 
@@ -922,9 +911,8 @@ <h2 id="raises">Raises</h2>
 
     Args:
         model_db_dir: Directory to store model data in. Defaults to DEFAULT_LOCAL_GGUF_DIR.
-        max_tokens: Max tokens to be generated by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
-        max_input_tokens: Max input tokens to be generated by LlamaAI model. (Defaults to 100, set to around 200-300 for regular use)
-
+        max_total_tokens: Max tokens to be processed (input+generation) by LlamaAI model. (Defaults to 200, set to around 500-1k for regular use)
+
         Provide at least one of these args to fetch ModelData: 
         ---
         model_url: URL of model to configure with. Automatically downloads and builds as needed. (Optional)
@@ -951,7 +939,7 @@ <h2 id="raises">Raises</h2>
     else:
         raise Exception(&#34;Can&#39;t find model data. Please provide a model URL, GGUF file path, or model name/quantization/keyword.&#34;)
 
-    self.load_ai(new_tokens, max_input_tokens)</code></pre>
+    self.load_ai(max_total_tokens)</code></pre>
 </details>
 </dd>
 <dt id="glai.ai.EasyAI.count_tokens"><code class="name flex">
@@ -1164,17 +1152,15 @@ <h2 id="returns">Returns</h2>
 </details>
 </dd>
 <dt id="glai.ai.EasyAI.load_ai"><code class="name flex">
-<span>def <span class="ident">load_ai</span></span>(<span>self, max_tokens: int = 200, max_input_tokens: int = 100) ‑> None</span>
+<span>def <span class="ident">load_ai</span></span>(<span>self, max_total_tokens: int = 200) ‑> None</span>
 </code></dt>
 <dd>
 <div class="desc"><p>Load LlamaAI model from model data.</p>
 <p>Downloads model file from model data URL if needed. Initializes LlamaAI with model and sets lai attribute.</p>
 <h2 id="args">Args</h2>
 <dl>
-<dt><strong><code>max_tokens</code></strong></dt>
+<dt><strong><code>max_total_tokens</code></strong></dt>
 <dd>Max tokens for LlamaAI model.</dd>
-<dt><strong><code>max_input_tokens</code></strong></dt>
-<dd>Max input tokens for LlamaAI model.</dd>
 </dl>
 <h2 id="raises">Raises</h2>
 <dl>
@@ -1186,17 +1172,14 @@ <h2 id="raises">Raises</h2>
 <span>Expand source code</span>
 </summary>
 <pre><code class="python">def load_ai(self,
-            max_tokens: int = 200,
-            max_input_tokens: int = 100) -&gt; None:
+            max_total_tokens: int = 200,) -&gt; None:
     &#34;&#34;&#34;
     Load LlamaAI model from model data.
 
     Downloads model file from model data URL if needed. Initializes LlamaAI with model and sets lai attribute.
 
     Args:
-        max_tokens: Max tokens for LlamaAI model.
-        max_input_tokens: Max input tokens for LlamaAI model.
-
+        max_total_tokens: Max tokens for LlamaAI model.
     Raises:
         Exception: If no model data or messages loaded yet.
     &#34;&#34;&#34;
@@ -1206,7 +1189,7 @@ <h2 id="raises">Raises</h2>
     if self.model_data is None:
         raise Exception(&#34;No model data loaded. Use find_model_data(), get_model_data_from_url(), or get_model_data_from_file() first.&#34;)
     self.model_data.download_gguf()
-    self.ai = LlamaAI(self.model_data.model_path(), max_tokens=max_tokens, max_input_tokens=max_input_tokens)
+    self.ai = LlamaAI(self.model_data.model_path(), max_tokens=max_total_tokens)
     print(f&#34;Loaded: {self.model_data}&#34;)</code></pre>
 </details>
 </dd>

diff --git a/docs/glai/back_end/index.html b/docs/glai/back_end/index.html
@@ -1008,10 +1008,10 @@ <h2 id="returns">Returns</h2>
             model_db_dir = MODEL_EXAMPLES_DB_DIR
         self.set_model_db_dir(model_db_dir)
         if model_db_dir != MODEL_EXAMPLES_DB_DIR:
-            print(f&#34;Copying examples to {model_db_dir}...&#34;)
             if copy_examples:
-                for file in list_files_in_dir(MODEL_EXAMPLES_DB_DIR, show_directories=False, only_with_extensions=[&#34;.json&#34;], absolute=False):
-                    f_mdt = ModelData.from_json(join_paths(MODEL_EXAMPLES_DB_DIR, file))
+                print(f&#34;Copying examples to {model_db_dir}...&#34;)
+                for file in list_files_in_dir(MODEL_EXAMPLES_DB_DIR, include_directories=False, only_with_extensions=[&#34;.json&#34;], just_names=False):
+                    f_mdt = ModelData.from_json(file)
                     f_mdt.set_save_dir(model_db_dir)
                     f_mdt.save_json()
                     print(f&#34;Saved a copy of {file} to {model_db_dir}.&#34;)
@@ -1025,14 +1025,13 @@ <h2 id="returns">Returns</h2>
 
     def load_models(self) -&gt; None:
         self.models = []
-        files = list_files_in_dir(self.gguf_db_dir, show_directories=False, show_files=True, only_with_extensions=[&#34;.json&#34;])
+        files = list_files_in_dir(self.gguf_db_dir, include_directories=False, include_files=True, only_with_extensions=[&#34;.json&#34;])
         for file in files:
             try:
-                file_path = join_paths(self.gguf_db_dir, file)
-                model_data = ModelData.from_json(file_path)
+                model_data = ModelData.from_json(file)
                 self.models.append(model_data)
             except Exception as e:
-                print(f&#34;Error trying to load from {file_path}: \t\n{e}, \nskipping...&#34;)
+                print(f&#34;Error trying to load from {file}: \t\n{e}, \nskipping...&#34;)
                 continue
         print(f&#34;Loaded {len(self.models)} models from {self.gguf_db_dir}.&#34;)
 
@@ -1419,14 +1418,13 @@ <h3>Methods</h3>
 </summary>
 <pre><code class="python">def load_models(self) -&gt; None:
     self.models = []
-    files = list_files_in_dir(self.gguf_db_dir, show_directories=False, show_files=True, only_with_extensions=[&#34;.json&#34;])
+    files = list_files_in_dir(self.gguf_db_dir, include_directories=False, include_files=True, only_with_extensions=[&#34;.json&#34;])
     for file in files:
         try:
-            file_path = join_paths(self.gguf_db_dir, file)
-            model_data = ModelData.from_json(file_path)
+            model_data = ModelData.from_json(file)
             self.models.append(model_data)
         except Exception as e:
-            print(f&#34;Error trying to load from {file_path}: \t\n{e}, \nskipping...&#34;)
+            print(f&#34;Error trying to load from {file}: \t\n{e}, \nskipping...&#34;)
             continue
     print(f&#34;Loaded {len(self.models)} models from {self.gguf_db_dir}.&#34;)</code></pre>
 </details>