deploy: 05f4e72

Marker-Inc-Korea · Dec 15, 2024 · 5daa05a · 5daa05a
1 parent e0c1ffc
commit 5daa05a
Show file tree

Hide file tree

Showing 12 changed files with 90 additions and 153 deletions.
diff --git a/.doctrees/api_spec/autorag.deploy.doctree b/.doctrees/api_spec/autorag.deploy.doctree
diff --git a/.doctrees/api_spec/autorag.doctree b/.doctrees/api_spec/autorag.doctree
diff --git a/.doctrees/deploy/api_endpoint.doctree b/.doctrees/deploy/api_endpoint.doctree
diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle
diff --git a/_modules/autorag/deploy/api.html b/_modules/autorag/deploy/api.html
@@ -437,6 +437,7 @@ <h1>Source code for autorag.deploy.api</h1><div class="highlight"><pre>
 <span class="k">class</span> <span class="nc">RetrievedPassage</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
 	<span class="n">content</span><span class="p">:</span> <span class="nb">str</span>
 	<span class="n">doc_id</span><span class="p">:</span> <span class="nb">str</span>
+	<span class="n">score</span><span class="p">:</span> <span class="nb">float</span>
 	<span class="n">filepath</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
 	<span class="n">file_page</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
 	<span class="n">start_idx</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
@@ -452,19 +453,10 @@ <h1>Source code for autorag.deploy.api</h1><div class="highlight"><pre>
 
 
 
-<div class="viewcode-block" id="Passage">
-<a class="viewcode-back" href="../../../api_spec/autorag.deploy.html#autorag.deploy.api.Passage">[docs]</a>
-<span class="k">class</span> <span class="nc">Passage</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
-	<span class="n">doc_id</span><span class="p">:</span> <span class="nb">str</span>
-	<span class="n">content</span><span class="p">:</span> <span class="nb">str</span>
-	<span class="n">score</span><span class="p">:</span> <span class="nb">float</span></div>
-
-
-
 <div class="viewcode-block" id="RetrievalResponse">
 <a class="viewcode-back" href="../../../api_spec/autorag.deploy.html#autorag.deploy.api.RetrievalResponse">[docs]</a>
 <span class="k">class</span> <span class="nc">RetrievalResponse</span><span class="p">(</span><span class="n">BaseModel</span><span class="p">):</span>
-	<span class="n">passages</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Passage</span><span class="p">]</span></div>
+	<span class="n">passages</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">RetrievedPassage</span><span class="p">]</span></div>
 
 
 
@@ -578,18 +570,9 @@ <h1>Source code for autorag.deploy.api</h1><div class="highlight"><pre>
 				<span class="n">previous_result</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">drop_previous_result</span><span class="p">,</span> <span class="n">new_result</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
 
 			<span class="c1"># Simulate processing the query</span>
-			<span class="n">retrieved_contents</span> <span class="o">=</span> <span class="n">previous_result</span><span class="p">[</span><span class="s2">&quot;retrieved_contents&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
-			<span class="n">retrieved_ids</span> <span class="o">=</span> <span class="n">previous_result</span><span class="p">[</span><span class="s2">&quot;retrieved_ids&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
-			<span class="n">retrieve_scores</span> <span class="o">=</span> <span class="n">previous_result</span><span class="p">[</span><span class="s2">&quot;retrieve_scores&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
-
-			<span class="n">retrieval_response</span> <span class="o">=</span> <span class="n">RetrievalResponse</span><span class="p">(</span>
-				<span class="n">passages</span><span class="o">=</span><span class="p">[</span>
-					<span class="n">Passage</span><span class="p">(</span><span class="n">doc_id</span><span class="o">=</span><span class="n">doc_id</span><span class="p">,</span> <span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span> <span class="n">score</span><span class="o">=</span><span class="n">score</span><span class="p">)</span>
-					<span class="k">for</span> <span class="n">doc_id</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">score</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span>
-						<span class="n">retrieved_ids</span><span class="p">,</span> <span class="n">retrieved_contents</span><span class="p">,</span> <span class="n">retrieve_scores</span>
-					<span class="p">)</span>
-				<span class="p">]</span>
-			<span class="p">)</span>
+			<span class="n">retrieved_passages</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">extract_retrieve_passage</span><span class="p">(</span><span class="n">previous_result</span><span class="p">)</span>
+
+			<span class="n">retrieval_response</span> <span class="o">=</span> <span class="n">RetrievalResponse</span><span class="p">(</span><span class="n">passages</span><span class="o">=</span><span class="n">retrieved_passages</span><span class="p">)</span>
 			<span class="k">return</span> <span class="n">jsonify</span><span class="p">(</span><span class="n">retrieval_response</span><span class="o">.</span><span class="n">model_dump</span><span class="p">()),</span> <span class="mi">200</span>
 
 		<span class="nd">@self</span><span class="o">.</span><span class="n">app</span><span class="o">.</span><span class="n">route</span><span class="p">(</span><span class="s2">&quot;/v1/stream&quot;</span><span class="p">,</span> <span class="n">methods</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;POST&quot;</span><span class="p">])</span>
@@ -694,6 +677,7 @@ <h1>Source code for autorag.deploy.api</h1><div class="highlight"><pre>
 	<span class="k">def</span> <span class="nf">extract_retrieve_passage</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">RetrievedPassage</span><span class="p">]:</span>
 		<span class="n">retrieved_ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s2">&quot;retrieved_ids&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
 		<span class="n">contents</span> <span class="o">=</span> <span class="n">fetch_contents</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">corpus_df</span><span class="p">,</span> <span class="p">[</span><span class="n">retrieved_ids</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
+		<span class="n">scores</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="s2">&quot;retrieve_scores&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
 		<span class="k">if</span> <span class="s2">&quot;path&quot;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">corpus_df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
 			<span class="n">paths</span> <span class="o">=</span> <span class="n">fetch_contents</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">corpus_df</span><span class="p">,</span> <span class="p">[</span><span class="n">retrieved_ids</span><span class="p">],</span> <span class="n">column_name</span><span class="o">=</span><span class="s2">&quot;path&quot;</span><span class="p">)[</span>
 				<span class="mi">0</span>
@@ -712,16 +696,23 @@ <h1>Source code for autorag.deploy.api</h1><div class="highlight"><pre>
 		<span class="n">start_end_indices</span> <span class="o">=</span> <span class="n">to_list</span><span class="p">(</span><span class="n">start_end_indices</span><span class="p">)</span>
 		<span class="k">return</span> <span class="nb">list</span><span class="p">(</span>
 			<span class="nb">map</span><span class="p">(</span>
-				<span class="k">lambda</span> <span class="n">content</span><span class="p">,</span> <span class="n">doc_id</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">metadata</span><span class="p">,</span> <span class="n">start_end_idx</span><span class="p">:</span> <span class="n">RetrievedPassage</span><span class="p">(</span>
+				<span class="k">lambda</span> <span class="n">content</span><span class="p">,</span>
+				<span class="n">doc_id</span><span class="p">,</span>
+				<span class="n">score</span><span class="p">,</span>
+				<span class="n">path</span><span class="p">,</span>
+				<span class="n">metadata</span><span class="p">,</span>
+				<span class="n">start_end_idx</span><span class="p">:</span> <span class="n">RetrievedPassage</span><span class="p">(</span>
 					<span class="n">content</span><span class="o">=</span><span class="n">content</span><span class="p">,</span>
 					<span class="n">doc_id</span><span class="o">=</span><span class="n">doc_id</span><span class="p">,</span>
+					<span class="n">score</span><span class="o">=</span><span class="n">score</span><span class="p">,</span>
 					<span class="n">filepath</span><span class="o">=</span><span class="n">path</span><span class="p">,</span>
 					<span class="n">file_page</span><span class="o">=</span><span class="n">metadata</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;page&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span>
 					<span class="n">start_idx</span><span class="o">=</span><span class="n">start_end_idx</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="n">start_end_idx</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
 					<span class="n">end_idx</span><span class="o">=</span><span class="n">start_end_idx</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="n">start_end_idx</span> <span class="k">else</span> <span class="kc">None</span><span class="p">,</span>
 				<span class="p">),</span>
 				<span class="n">contents</span><span class="p">,</span>
 				<span class="n">retrieved_ids</span><span class="p">,</span>
+				<span class="n">scores</span><span class="p">,</span>
 				<span class="n">paths</span><span class="p">,</span>
 				<span class="n">metadatas</span><span class="p">,</span>
 				<span class="n">start_end_indices</span><span class="p">,</span>

diff --git a/_sources/deploy/api_endpoint.md.txt b/_sources/deploy/api_endpoint.md.txt
@@ -72,6 +72,7 @@ This is the URL to your local server, so use it as the host at request.
           - **Properties**:
             - `content` (string): The content of the passage.
             - `doc_id` (string): Document ID.
+            - `score` (string): The relevance score of the retrieved passage.
             - `filepath` (string, nullable): File path.
             - `file_page` (integer, nullable): File page number.
             - `start_idx` (integer, nullable): Start index.
@@ -105,31 +106,17 @@ The request must include a JSON object with the following structure:
 #### Success Response
 **HTTP Status Code:** `200 OK`
 
-#### Response Body
-On a successful retrieval, the response will contain a JSON object structured as follows:
-
-```json
-{
-  "passages": [
-    {
-      "doc_id": "unique-document-id-1",
-      "content": "Content of the retrieved document.",
-      "score": 0.95
-    },
-    {
-      "doc_id": "unique-document-id-2",
-      "content": "Content of another retrieved document.",
-      "score": 0.89
-    }
-  ]
-}
-```
+### Response Body
 
 #### Properties
 - **passages** (array): An array of documents retrieved based on the query.
   - **doc_id** (string): The unique identifier for each document.
   - **content** (string): The content of the retrieved document.
   - **score** (number, float): The relevance score of the retrieved document.
+  - **filepath** (string, optional): The file path of the document.
+  - **file_page** (integer, optional): The page number of the document.
+  - **start_idx** (integer, optional): The start index of the retrieved passage from the parsed data.
+  - **end_idx** (integer, optional): The end index of the retrieved passage from the parsed data.
 
 #### Example Response
 ```json
@@ -138,12 +125,20 @@ On a successful retrieval, the response will contain a JSON object structured as
     {
       "doc_id": "doc123",
       "content": "Artificial Intelligence is transforming industries.",
-      "score": 0.98
+      "score": 0.98,
+      "filepath": "path/to/file",
+      "file_page": 2,
+      "start_idx": 100,
+      "end_idx": 150
     },
     {
       "doc_id": "doc456",
       "content": "The future of AI includes advancements in machine learning.",
-      "score": 0.92
+      "score": 0.92,
+      "filepath": null,
+      "file_page": null,
+      "start_idx": null,
+      "end_idx": null
     }
   ]
 }
@@ -172,6 +167,7 @@ On a successful retrieval, the response will contain a JSON object structured as
           - **Properties**:
             - `content` (string): The content of the passage.
             - `doc_id` (string): Document ID.
+            - `score` (string): The relevance score of the retrieved passage.
             - `filepath` (string, nullable): File path.
             - `file_page` (integer, nullable): File page number.
             - `start_idx` (integer, nullable): Start index.
@@ -295,6 +291,14 @@ curl -X POST "http://example.com/v1/stream" \
      --no-buffer
 ```
 
+### `/v1/retrieve` (POST)
+
+```bash
+curl -X POST "http://example.com/v1/retrieve" \
+     -H "Content-Type: application/json" \
+     -d '{"query": "example query"}'
+```
+
 #### `/version` (GET)
 
 ```bash