lambci as a base image

Author: Valian

Dockerfile

@@ -1,10 +1,23 @@
-FROM python:3.6
+FROM python:3.6 as builder
 
-WORKDIR /srv
 
 COPY requirements.txt requirements.txt
 RUN pip install -r requirements.txt
 
-COPY . .
+COPY . /srv
+
+
+FROM lambci/lambda:python3.6
+
+WORKDIR /srv
+
+USER root
+
+RUN yum install -y git
+
+COPY --from=builder /usr/local/lib/python3.6/site-packages /var/lang/lib/python3.6/site-packages
+COPY --from=builder /srv /srv
+
 
+ENTRYPOINT []
 CMD ["python", "-u", "app.py"]

app.py

@@ -1,80 +1,44 @@
 import argparse
 import asyncio
 import hashlib
-import importlib
 import json
 import shutil
 import subprocess
 import sys
+import os
 
 import traceback
-import timeit
 import uuid
 
 import logging
 
-import math
 from functools import partial
 
 from aiohttp import web
 from os import path
 
-from lambda_local.main import load_lib
-from lambda_local.context import Context
 
-SOURCES_DIR = "/src"
+SOURCES_DIR = "/var/task"
 SOURCES_REQUIREMENTS_NAME = path.join(SOURCES_DIR, "requirements.txt")
 PACKAGES_DIR = "/packages"
 PACKAGES_REQUIREMENTS_PATH = path.join(PACKAGES_DIR, "requirements.txt")
+LAMBDA_USER_ID = 496
 
 logger = logging.getLogger('lambda')
 
 
+def demote(user_uid, user_gid):
+    os.setgid(user_gid)
+    os.setuid(user_uid)
+
+
 def jsonify(data, status_code=200):
     return web.Response(
         text=json.dumps(data),
         headers={'Content-Type': 'application/json'},
         status=status_code)
 
 
-import os
-import types
-import importlib
-
-
-def reload_package(package):
-    assert(hasattr(package, "__package__"))
-    fn = package.__file__
-    fn_dir = os.path.dirname(fn) + os.sep
-    module_visit = {fn}
-    del fn
-
-    def reload_recursive_ex(module):
-        importlib.reload(module)
-
-        for module_child in vars(module).values():
-            if isinstance(module_child, types.ModuleType):
-                fn_child = getattr(module_child, "__file__", None)
-                if (fn_child is not None) and fn_child.startswith(fn_dir):
-                    if fn_child not in module_visit:
-                        # print("reloading:", fn_child, "from", module)
-                        module_visit.add(fn_child)
-                        reload_recursive_ex(module_child)
-
-    return reload_recursive_ex(package)
-
-
-def load(directory, module, handler_path):
-    file_path = path.join(directory, module)
-    file_directory = path.dirname(file_path)
-    sys.path.append(file_directory)
-    function_file, function_name = path.splitext(handler_path)
-    mod = importlib.import_module(function_file)
-    reload_package(mod)
-    func = getattr(mod, function_name.strip('.'))
-    return func
-
-
 async def parse_request(args, request):
     data = await request.json()
     arn_string = data.get('arn', '')
@@ -86,53 +50,30 @@ async def parse_request(args, request):
     if isinstance(event, str):
         event = json.loads(event)
 
-    context = Context(args.timeout, arn_string, version)
-    func = load(args.directory, module, handler)
-    return func, event, context
+    return handler, event
 
 
-async def execute(func, event, context):
-    loop = asyncio.get_event_loop()
-    try:
-        future = loop.run_in_executor(None, func, event, context)
-        result = await asyncio.wait_for(future, context.timeout)
-    except asyncio.TimeoutError as e:
-        result = e
-    except Exception:
-        err = sys.exc_info()
-        result = json.dumps({
-            "errorMessage": str(err[1]),
-            "stackTrace": traceback.format_tb(err[2]),
-            "errorType": err[0].__name__
-        }, indent=4, separators=(',', ': '))
-    return result
-
-
-async def async_execute(request_id, func, event, context):
-    logger.info("Event: {}".format(event))
-    logger.info("START RequestId: {}".format(request_id))
-
-    start_time = timeit.default_timer()
-    result = await execute(func, event, context)
-    end_time = timeit.default_timer()
-
-    logger.info("END RequestId: {}".format(request_id))
-
-    output_func = logger.error if type(result) is Exception else logger.info
-    output_func(f"RESULT:\n{result}")
-    duration = (end_time - start_time) * 1000
-    billed_duration = math.ceil(duration / 100) * 100
-    logger.info(
-        f"REPORT RequestId: {request_id}\t"
-        f"Duration: {duration:.2f} ms\t"
-        f"Billed Duration: {billed_duration:.2f} ms")
-    return result
+async def async_execute(handler, event):
+    process = await asyncio.create_subprocess_exec(
+        "python", "bootstrap.py", handler, json.dumps(event),
+        cwd='/var/runtime/awslambda/',
+        env={**os.environ, 'PYTHONPATH': PACKAGES_DIR},
+        preexec_fn=partial(demote, LAMBDA_USER_ID, LAMBDA_USER_ID),
+        stdin=asyncio.subprocess.PIPE,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    stdout, stderr = await process.communicate()
+    if stdout:
+        stdout = json.loads(stdout.decode('utf-8'))
+    if stderr:
+        stderr = stderr.decode('utf-8')
+    return {'stdout': stdout, 'stderr': stderr}
 
 
 async def run_lambda(args, request):
-    func, event, context = await parse_request(args, request)
-    request_id = uuid.uuid4()
-    result = await async_execute(request_id, func, event, context)
+    handler, event = await parse_request(args, request)
+    result = await async_execute(handler, event)
     return jsonify(data=result)
 
 
@@ -162,7 +103,6 @@ def install_requirements(args, force=False):
         shutil.copy(requirements_path, previous_requirements_path)
     else:
         logger.info("Requirements not changed, skipping update...")
-    load_lib(PACKAGES_DIR)
 
 
 def init_logging(args):

docker-compose.yml

@@ -6,4 +6,4 @@ services:
       - 8080:8080
     volumes:
       - .:/srv
-      - ./test:/src/
+      - ./test:/var/task

index.html

@@ -75,7 +75,14 @@ <h1 class="subtitle">Output</h1>
         <div v-if="loading">Loading...</div>
         <div v-else>
           <div v-if="tab === 'event'">
-            {{ this.output }}
+            <ul>
+              <li>
+                <strong>Output:</strong><pre>{{ this.responseData }}</pre>
+              </li>
+              <li>
+                <strong>Logs:</strong><pre>{{ this.responseLog }}</pre>
+              </li>
+            </ul>
           </div>
           <div v-else>
             <ul>
@@ -89,6 +96,9 @@ <h1 class="subtitle">Output</h1>
                 <strong>Body:</strong>
                 <pre>{{ this.responseBody | prettify }}</pre>
               </li>
+              <li>
+                <strong>Logs:</strong><pre>{{ this.responseLog }}</pre>
+              </li>
             </ul>
           </div>
         </div>
@@ -122,21 +132,24 @@ <h1 class="subtitle">Output</h1>
       this.loadState()
     },
     computed: {
-      isJsonResponse: function () {
-        return this.output && this.output.headers && this.output.headers['Content-Type'] === 'application/json'
+      responseData: function () {
+        return this.output && this.output.stdout
       },
-      responseBody: function () {
-        if(this.isJsonResponse) {
-          return JSON.parse(this.output.body)
-        } else {
-          return this.output && this.output.body
-        }
+      responseLog: function () {
+        return this.output && this.output.stderr
       },
       responseStatusCode: function () {
-        return this.output && this.output.statusCode
+        return this.responseData && this.responseData.statusCode
       },
       responseHeaders: function () {
-        return this.output && this.output.headers
+        return this.responseData && this.responseData.headers
+      },
+      responseBody: function () {
+        try {
+          return JSON.parse(this.responseData.body)
+        } catch(e) {
+          return this.responseData && this.responseData.body
+        }
       }
     },
     filters: {

requirements.in

@@ -1,2 +1 @@
-aiohttp
-python-lambda-local
+aiohttp

requirements.txt

@@ -7,16 +7,8 @@
 aiohttp==3.1.1
 async-timeout==2.0.1      # via aiohttp
 attrs==17.4.0             # via aiohttp
-boto3==1.6.17             # via python-lambda-local
-botocore==1.9.17          # via boto3, s3transfer
 chardet==3.0.4            # via aiohttp
-docutils==0.14            # via botocore
 idna-ssl==1.0.1           # via aiohttp
 idna==2.6                 # via idna-ssl, yarl
-jmespath==0.9.3           # via boto3, botocore
 multidict==4.1.0          # via aiohttp, yarl
-python-dateutil==2.6.1    # via botocore
-python-lambda-local==0.1.5
-s3transfer==0.1.13        # via boto3
-six==1.11.0               # via python-dateutil
 yarl==1.1.1               # via aiohttp

test/handler.py

@@ -4,6 +4,6 @@
 def handler(event, context):
     response = requests.get('http://example.com')
     return {
-        "statusCode": response.status_code,
+        "statusCode": 200,
         "url": response.url
     }