[Fix] Unified LLM Whisperer adapters

src/unstract/sdk/adapters/x2text/llm_whisperer/README.md

@@ -4,7 +4,55 @@
 
 The below env variables are resolved by LLMWhisperer adapter
 
-| Variable                     | Description                                                                                  |
-| ---------------------------- | -------------------------------------------------------------------------------------------- |
-| `ADAPTER_LLMW_POLL_INTERVAL` | Time in seconds to wait before polling LLMWhisperer's status API. Defaults to 30s            |
-| `ADAPTER_LLMW_MAX_POLLS`     | Total number of times to poll the status API. Defaults to 30                                 |
+| Variable                     | Description                                                                       |
+| ---------------------------- | --------------------------------------------------------------------------------- |
+| `ADAPTER_LLMW_POLL_INTERVAL` | Time in seconds to wait before polling LLMWhisperer's status API. Defaults to 30s |
+| `ADAPTER_LLMW_MAX_POLLS`     | Total number of times to poll the status API. Defaults to 30                      |
+
+---
+
+## id: llm_whisperer_apis_changelog
+
+# Changelog
+
+## Version 2.0.0
+
+:::warning
+This version of the API is not backward compatible with the previous version.
+:::
+
+### API endpoint
+
+- The base URL for the **V2** APIs is `https://llmwhisperer-api.unstract.com/api/v2`
+
+### Global change in parameter naming
+
+- All use of `whisper-hash` as a parameter has been replaced with `whisper_hash` for consistency.
+
+### Whisper parameters
+
+#### Added
+
+- `mode` (str, optional): The processing mode.
+- `mark_vertical_lines` (bool, optional): Whether to reproduce vertical lines in the document.
+- `mark_horizontal_lines` (bool, optional): Whether to reproduce horizontal lines in the document.
+- `line_splitter_strategy` (str, optional): The line splitter strategy to use. An advanced option for customizing the line splitting process.
+- `lang` (str, optional): The language of the document.
+- `tag` (str, optional): A tag to associate with the document. Used for auditing and tracking purposes.
+- `file_name` (str, optional): The name of the file being processed. Used for auditing and tracking purposes.
+- `use_webhook` (str, optional): The name of the webhook to call after the document is processed.
+- `webhook_metadata` (str, optional): Metadata to send to the webhook after the document is processed.
+
+#### Removed
+
+- `timeout` (int, optional): The timeout for API requests. _There is no sync mode now. All requests are async._
+- `force_text_processing` (bool, optional): Whether to force text processing. _This is feature is removed_
+- `ocr_provider` (str, optional): The OCR provider to use. _This is superseded by `mode`_
+- `processing_mode` (str, optional): The processing mode. _This is superseded by `mode`_
+- `store_metadata_for_highlighting` (bool, optional): Whether to store metadata for highlighting. _Feature is removed. Data still available and set back when retrieve is called_
+
+### New features
+
+#### Webhooks
+
+- Added support for webhooks. You can now register a webhook and use it to receive the processed document.

src/unstract/sdk/adapters/x2text/llm_whisperer/src/constants.py

@@ -18,6 +18,7 @@ class OutputModes(Enum):
     LINE_PRINTER = "line-printer"
     DUMP_TEXT = "dump-text"
     TEXT = "text"
+    LAYOUT_PRESERVING = "layout_preserving"
 
 
 class HTTPMethod(Enum):
@@ -48,10 +49,15 @@ class WhispererEnv:
             LLMWhisperer's status API. Defaults to 30s
         MAX_POLLS: Total number of times to poll the status API.
             Set to -1 to poll indefinitely. Defaults to -1
+        STATUS_RETRIES: Number of times to retry calling LLLMWhisperer's status API
+            on failure during polling. Defaults to 5.
     """
 
     POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL"
     MAX_POLLS = "ADAPTER_LLMW_MAX_POLLS"
+    POLL_INTERVAL_V2 = "ADAPTER_LLMW_POLL_INTERVAL_V2"
+    MAX_POLLS_V2 = "ADAPTER_LLMW_MAX_POLLS_V2"
+    STATUS_RETRIES = "ADAPTER_LLMW_STATUS_RETRIES"
 
 
 class WhispererConfig:
@@ -66,6 +72,7 @@ class WhispererConfig:
     GAUSSIAN_BLUR_RADIUS = "gaussian_blur_radius"
     FORCE_TEXT_PROCESSING = "force_text_processing"
     LINE_SPLITTER_TOLERANCE = "line_splitter_tolerance"
+    LINE_SPLITTER_STRATEGY = "line_splitter_strategy"
     HORIZONTAL_STRETCH_FACTOR = "horizontal_stretch_factor"
     PAGES_TO_EXTRACT = "pages_to_extract"
     STORE_METADATA_FOR_HIGHLIGHTING = "store_metadata_for_highlighting"
@@ -74,7 +81,12 @@ class WhispererConfig:
     PAGE_SEPARATOR = "page_seperator"
     MARK_VERTICAL_LINES = "mark_vertical_lines"
     MARK_HORIZONTAL_LINES = "mark_horizontal_lines"
-
+    URL_IN_POST = "url_in_post"
+    TAG = "tag"
+    USE_WEBHOOK = "use_webhook"
+    WEBHOOK_METADATA = "webhook_metadata"
+    TEXT_ONLY = "text_only"
+    VERSION = "version"
 
 class WhisperStatus:
     """Values returned / used by /whisper-status endpoint."""
@@ -86,6 +98,7 @@ class WhisperStatus:
     # Used for async processing
     WHISPER_HASH = "whisper-hash"
     STATUS = "status"
+    WHISPER_HASH_V2 = "whisper_hash"
 
 
 class WhispererDefaults:
@@ -95,12 +108,19 @@ class WhispererDefaults:
     GAUSSIAN_BLUR_RADIUS = 0.0
     FORCE_TEXT_PROCESSING = False
     LINE_SPLITTER_TOLERANCE = 0.75
+    LINE_SPLITTER_STRATEGY = "left-priority"
     HORIZONTAL_STRETCH_FACTOR = 1.0
     POLL_INTERVAL = int(os.getenv(WhispererEnv.POLL_INTERVAL, 30))
     MAX_POLLS = int(os.getenv(WhispererEnv.MAX_POLLS, 30))
+    POLL_INTERVAL_V2 = int(os.getenv(WhispererEnv.POLL_INTERVAL_V2, 30))
+    MAX_POLLS_V2 = int(os.getenv(WhispererEnv.MAX_POLLS_V2, 30))
     PAGES_TO_EXTRACT = ""
     ADD_LINE_NOS = True
     OUTPUT_JSON = True
     PAGE_SEPARATOR = "<<< >>>"
     MARK_VERTICAL_LINES = False
     MARK_HORIZONTAL_LINES = False
+    STATUS_RETRIES = int(os.getenv(WhispererEnv.STATUS_RETRIES, 5))
+    URL_IN_POST = False
+    TAG = "default"
+    TEXT_ONLY = False

src/unstract/sdk/adapters/x2text/llm_whisperer/src/helper.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any
+from unstract.sdk.adapters.x2text.llm_whisperer.src.constants import (
+    Modes,
+    OutputModes,
+    WhispererConfig,
+    WhispererDefaults,
+)
+logger = logging.getLogger(__name__)
+
+
+class LLMWhispererHelper:
+
+    @staticmethod
+    def get_whisperer_params(config: dict[str, Any]) -> dict[str, Any]:
+        """Gets query params meant for /whisper endpoint.
+
+        The params is filled based on the configuration passed.
+
+        Returns:
+            dict[str, Any]: Query params
+        """
+        params = {
+            WhispererConfig.MODE: config.get(WhispererConfig.MODE, Modes.FORM.value),
+            WhispererConfig.OUTPUT_MODE: config.get(
+                WhispererConfig.OUTPUT_MODE, OutputModes.LAYOUT_PRESERVING.value
+            ),
+            WhispererConfig.LINE_SPLITTER_TOLERANCE: config.get(
+                WhispererConfig.LINE_SPLITTER_TOLERANCE,
+                WhispererDefaults.LINE_SPLITTER_TOLERANCE,
+            ),
+            WhispererConfig.LINE_SPLITTER_STRATEGY: config.get(
+                WhispererConfig.LINE_SPLITTER_STRATEGY,
+                WhispererDefaults.LINE_SPLITTER_STRATEGY,
+            ),
+            WhispererConfig.HORIZONTAL_STRETCH_FACTOR: config.get(
+                WhispererConfig.HORIZONTAL_STRETCH_FACTOR,
+                WhispererDefaults.HORIZONTAL_STRETCH_FACTOR,
+            ),
+            WhispererConfig.PAGES_TO_EXTRACT: config.get(
+                WhispererConfig.PAGES_TO_EXTRACT,
+                WhispererDefaults.PAGES_TO_EXTRACT,
+            ),
+            WhispererConfig.MARK_VERTICAL_LINES: config.get(
+                WhispererConfig.MARK_VERTICAL_LINES,
+                WhispererDefaults.MARK_VERTICAL_LINES,
+            ),
+            WhispererConfig.MARK_HORIZONTAL_LINES: config.get(
+                WhispererConfig.MARK_HORIZONTAL_LINES,
+                WhispererDefaults.MARK_HORIZONTAL_LINES,
+            ),
+            WhispererConfig.URL_IN_POST: WhispererDefaults.URL_IN_POST,
+            WhispererConfig.PAGE_SEPARATOR: config.get(
+                WhispererConfig.PAGE_SEPARATOR,
+                WhispererDefaults.PAGE_SEPARATOR,
+            ),
+            # Not providing default value to maintain legacy compatablity
+            # these are optional params and identifiers for audit
+            WhispererConfig.TAG: config.get(
+                WhispererConfig.TAG,
+                WhispererDefaults.TAG,
+            ),
+            WhispererConfig.USE_WEBHOOK: config.get(WhispererConfig.USE_WEBHOOK),
+            WhispererConfig.WEBHOOK_METADATA: config.get(
+                WhispererConfig.WEBHOOK_METADATA
+            ),
+        }
+        if params[WhispererConfig.MODE] == Modes.LOW_COST.value:
+            params.update(
+                {
+                    WhispererConfig.MEDIAN_FILTER_SIZE: config.get(
+                        WhispererConfig.MEDIAN_FILTER_SIZE,
+                        WhispererDefaults.MEDIAN_FILTER_SIZE,
+                    ),
+                    WhispererConfig.GAUSSIAN_BLUR_RADIUS: config.get(
+                        WhispererConfig.GAUSSIAN_BLUR_RADIUS,
+                        WhispererDefaults.GAUSSIAN_BLUR_RADIUS,
+                    ),
+                }
+            )
+        return params