[Fix] Unified LLM Whisperer adapters

src/unstract/sdk/adapters/x2text/llm_whisperer/src/constants.py

@@ -18,6 +18,7 @@ class OutputModes(Enum):
     LINE_PRINTER = "line-printer"
     DUMP_TEXT = "dump-text"
     TEXT = "text"
+    LAYOUT_PRESERVING = "layout_preserving"
 
 
 class HTTPMethod(Enum):
@@ -48,10 +49,13 @@ class WhispererEnv:
             LLMWhisperer's status API. Defaults to 30s
         MAX_POLLS: Total number of times to poll the status API.
             Set to -1 to poll indefinitely. Defaults to -1
+        STATUS_RETRIES: Number of times to retry calling LLLMWhisperer's status API
+            on failure during polling. Defaults to 5.
     """
 
     POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL"
     MAX_POLLS = "ADAPTER_LLMW_MAX_POLLS"
+    STATUS_RETRIES = "ADAPTER_LLMW_STATUS_RETRIES"
 
 
 class WhispererConfig:
@@ -66,6 +70,7 @@ class WhispererConfig:
     GAUSSIAN_BLUR_RADIUS = "gaussian_blur_radius"
     FORCE_TEXT_PROCESSING = "force_text_processing"
     LINE_SPLITTER_TOLERANCE = "line_splitter_tolerance"
+    LINE_SPLITTER_STRATEGY = "line_splitter_strategy"
     HORIZONTAL_STRETCH_FACTOR = "horizontal_stretch_factor"
     PAGES_TO_EXTRACT = "pages_to_extract"
     STORE_METADATA_FOR_HIGHLIGHTING = "store_metadata_for_highlighting"
@@ -74,7 +79,12 @@ class WhispererConfig:
     PAGE_SEPARATOR = "page_seperator"
     MARK_VERTICAL_LINES = "mark_vertical_lines"
     MARK_HORIZONTAL_LINES = "mark_horizontal_lines"
-
+    URL_IN_POST = "url_in_post"
+    TAG = "tag"
+    USE_WEBHOOK = "use_webhook"
+    WEBHOOK_METADATA = "webhook_metadata"
+    TEXT_ONLY = "text_only"
+    VERSION = "version"
 
 class WhisperStatus:
     """Values returned / used by /whisper-status endpoint."""
@@ -86,6 +96,7 @@ class WhisperStatus:
     # Used for async processing
     WHISPER_HASH = "whisper-hash"
     STATUS = "status"
+    WHISPER_HASH_V2 = "whisper_hash"
 
 
 class WhispererDefaults:
@@ -95,6 +106,7 @@ class WhispererDefaults:
     GAUSSIAN_BLUR_RADIUS = 0.0
     FORCE_TEXT_PROCESSING = False
     LINE_SPLITTER_TOLERANCE = 0.75
+    LINE_SPLITTER_STRATEGY = "left-priority"
     HORIZONTAL_STRETCH_FACTOR = 1.0
     POLL_INTERVAL = int(os.getenv(WhispererEnv.POLL_INTERVAL, 30))
     MAX_POLLS = int(os.getenv(WhispererEnv.MAX_POLLS, 30))
@@ -104,3 +116,7 @@ class WhispererDefaults:
     PAGE_SEPARATOR = "<<< >>>"
     MARK_VERTICAL_LINES = False
     MARK_HORIZONTAL_LINES = False
+    STATUS_RETRIES = int(os.getenv(WhispererEnv.STATUS_RETRIES, 5))
+    URL_IN_POST = False
+    TAG = "default"
+    TEXT_ONLY = False

src/unstract/sdk/adapters/x2text/llm_whisperer/src/helper.py

@@ -0,0 +1,81 @@
+import logging
+from typing import Any
+from unstract.sdk.adapters.x2text.llm_whisperer.src.constants import (
+    Modes,
+    OutputModes,
+    WhispererConfig,
+    WhispererDefaults,
+)
+logger = logging.getLogger(__name__)
+
+
+class LLMWhispererHelper:
+
+    @staticmethod
+    def get_whisperer_params(config: dict[str, Any]) -> dict[str, Any]:
+        """Gets query params meant for /whisper endpoint.
+
+        The params is filled based on the configuration passed.
+
+        Returns:
+            dict[str, Any]: Query params
+        """
+        params = {
+            WhispererConfig.MODE: config.get(WhispererConfig.MODE, Modes.FORM.value),
+            WhispererConfig.OUTPUT_MODE: config.get(
+                WhispererConfig.OUTPUT_MODE, OutputModes.LAYOUT_PRESERVING.value
+            ),
+            WhispererConfig.LINE_SPLITTER_TOLERANCE: config.get(
+                WhispererConfig.LINE_SPLITTER_TOLERANCE,
+                WhispererDefaults.LINE_SPLITTER_TOLERANCE,
+            ),
+            WhispererConfig.LINE_SPLITTER_STRATEGY: config.get(
+                WhispererConfig.LINE_SPLITTER_STRATEGY,
+                WhispererDefaults.LINE_SPLITTER_STRATEGY,
+            ),
+            WhispererConfig.HORIZONTAL_STRETCH_FACTOR: config.get(
+                WhispererConfig.HORIZONTAL_STRETCH_FACTOR,
+                WhispererDefaults.HORIZONTAL_STRETCH_FACTOR,
+            ),
+            WhispererConfig.PAGES_TO_EXTRACT: config.get(
+                WhispererConfig.PAGES_TO_EXTRACT,
+                WhispererDefaults.PAGES_TO_EXTRACT,
+            ),
+            WhispererConfig.MARK_VERTICAL_LINES: config.get(
+                WhispererConfig.MARK_VERTICAL_LINES,
+                WhispererDefaults.MARK_VERTICAL_LINES,
+            ),
+            WhispererConfig.MARK_HORIZONTAL_LINES: config.get(
+                WhispererConfig.MARK_HORIZONTAL_LINES,
+                WhispererDefaults.MARK_HORIZONTAL_LINES,
+            ),
+            WhispererConfig.URL_IN_POST: WhispererDefaults.URL_IN_POST,
+            WhispererConfig.PAGE_SEPARATOR: config.get(
+                WhispererConfig.PAGE_SEPARATOR,
+                WhispererDefaults.PAGE_SEPARATOR,
+            ),
+            # Not providing default value to maintain legacy compatablity
+            # these are optional params and identifiers for audit
+            WhispererConfig.TAG: config.get(
+                WhispererConfig.TAG,
+                WhispererDefaults.TAG,
+            ),
+            WhispererConfig.USE_WEBHOOK: config.get(WhispererConfig.USE_WEBHOOK),
+            WhispererConfig.WEBHOOK_METADATA: config.get(
+                WhispererConfig.WEBHOOK_METADATA
+            ),
+        }
+        if params[WhispererConfig.MODE] == Modes.LOW_COST.value:
+            params.update(
+                {
+                    WhispererConfig.MEDIAN_FILTER_SIZE: config.get(
+                        WhispererConfig.MEDIAN_FILTER_SIZE,
+                        WhispererDefaults.MEDIAN_FILTER_SIZE,
+                    ),
+                    WhispererConfig.GAUSSIAN_BLUR_RADIUS: config.get(
+                        WhispererConfig.GAUSSIAN_BLUR_RADIUS,
+                        WhispererDefaults.GAUSSIAN_BLUR_RADIUS,
+                    ),
+                }
+            )
+        return params