SAAS-support-tasklogger-streams-2 (#33)

* - fixed some tests
- fixed githubs matching API
- cosmetics

* support task logger stream api

* fixed step logger options reference

* revert Dockerfile unwanted commits

* fixed write after end error

* change task logger package to use stream branch version

* dummy commit

* updated yarn.lock

* fixed tests

* wip

* Upgraded task logger to 1.5.2

* Upgraded task logger to 1.5.3

* dummy

* dummy

* Aligned branches to new paging structure

* dummy

Author: noamt-codefresh

Dockerfile

@@ -22,4 +22,3 @@ COPY . ./
 ENTRYPOINT ["/sbin/tini", "--"]
 
 CMD ["node",  "node_modules/.bin/forever", "--minUptime",  "1", "--spinSleepTime", "1000", "-c", "node", "lib/index.js"]
-

README.md

@@ -11,4 +11,6 @@
     # logging strategies
         # io.codefresh.loggerStrategy
             # logs - will get all container logs from the beginning
-            # attach - will get all container logs from the point where attach was enabled. usefull for getting all interactive i/o
+            # attach - will get all container logs from the point where attach was enabled. usefull for getting all interactive i/o
+            
+            

lib/ContainerLogger.js

@@ -5,6 +5,7 @@ const Q              = require('q');
 const logger         = require('cf-logs').Logger('codefresh:containerLogger');
 const CFError        = require('cf-errors');
 const LoggerStrategy = require('./enums').LoggerStrategy;
+const { Transform, pipeline } = require('stream');
 
 class ContainerLogger extends EventEmitter {
 
@@ -42,27 +43,25 @@ class ContainerLogger extends EventEmitter {
             })
             .then(([stdout, stderr]) => {
                 logger.info(`Attached stream to container: ${this.containerId}`);
+
                 // Listening on the stream needs to be performed different depending if a tty is attached or not
                 // See documentation of the docker api here: https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/attach-to-a-container
                 if (this.tty) {
-                    this._handleTtyStream(stdout, false);
-                    if (stderr) {
-                        this._handleTtyStream(stderr, true);
-                    }
-                } else {
-                    this._handleNonTtyStream(stdout, false);
-                }
-
-                stdout.on('end', () => {
-                    this.stepFinished = true;
-                    logger.info(`stdout end event was fired for container: ${this.containerId}`);
-                });
 
-                if (stderr) {
-                    stderr.on('end', () => {
+                    stdout.on('end', () => {
                         this.stepFinished = true;
-                        logger.info(`stderr end event was fired for container: ${this.containerId}`);
+                        logger.info(`stdout end event was fired for container: ${this.containerId}`);
                     });
+
+                    if (this.stepLogger.opts && this.stepLogger.opts.logsRateLimitConfig) {
+                        logger.info(`Found logger rate limit configuration, using streams api`);
+                        this._streamTty(stdout, stderr);
+                        return;
+                    }
+
+                    this._registerToTtyStreams(stdout, stderr);
+                } else {
+                    this._handleNonTtyStream(stdout, false);
                 }
             }, (err) => {
                 return Q.reject(new CFError({
@@ -99,6 +98,47 @@ class ContainerLogger extends EventEmitter {
         ]);
     }
 
+    _streamTty(stdout, stderr) {
+
+        pipeline(stdout, this._logSizeLimitStream(), this.stepLogger.writeStream(), (err) => {
+            if (err) {
+                logger.error(`Stdout streams pipeline failed on: ${err}`);
+                return;
+            }
+            logger.info('Stdout streams pipeline succeeded.');
+        });
+
+        if (!stderr) {
+            return;
+        }
+
+        pipeline(stderr, this._logSizeLimitStream(), this._errorTransformerStream(), this.stepLogger.writeStream(), (err) => {
+            if (err) {
+                logger.error(`Stderr streams pipeline failed on: ${err}`);
+                return;
+            }
+            logger.info('Stderr streams pipeline succeeded.');
+        });
+
+        stderr.once('end', () => {
+            this.stepFinished = true;
+            logger.info(`stderr end event was fired for container: ${this.containerId}`);
+        });
+
+    }
+
+    _registerToTtyStreams(stdout, stderr) {
+        this._handleTtyStream(stdout, false);
+
+        if (stderr) {
+            stderr.once('end', () => {
+                this.stepFinished = true;
+                logger.info(`stderr end event was fired for container: ${this.containerId}`);
+            });
+            this._handleTtyStream(stderr, true);
+        }
+    }
+
     _handleTtyStream(stream, isError) {
         stream.on('data', (chunk) => {
             const buf     = new Buffer(chunk);
@@ -150,6 +190,41 @@ class ContainerLogger extends EventEmitter {
         this.emit('message.logged');
     }
 
+    _errorTransformerStream() {
+        return new Transform({
+            transform: (data, encoding, done) => {
+                const message = `\x1B[31m${data.toString('utf8')}\x1B[0m`;
+                done(null, Buffer.from(message));
+            }
+        });
+    }
+
+    _logSizeLimitStream() {
+        return new Transform({
+            transform: (data, encoding, done) => {
+                if (this.logSizeLimit && (this._stepLogSizeExceeded() || this.isWorkflowLogSizeExceeded())) {
+                    if (!this.logExceededLimitsNotified) {
+                        this.logExceededLimitsNotified = true;
+                        const message = `\x1B[01;93mLog size exceeded for ${this._stepLogSizeExceeded() ? 'this step' : 'the workflow'}.\nThe step will continue to execute until it finished but new logs will not be stored.\x1B[0m\r\n`;
+                        done(null, Buffer.from(message));
+                        return;
+                    }
+
+                    done(null, Buffer.alloc(0));  // discard chunk
+                    return;
+                }
+
+                if (this.logSizeLimit) {
+                    this.logSize += Buffer.byteLength(data);
+                    this.stepLogger.setLogSize(this.logSize);
+                }
+
+                this.emit('message.logged');
+                done(null, data);
+            }
+        });
+    }
+
 }
 
 module.exports = ContainerLogger;

package.json

@@ -1,12 +1,12 @@
 {
   "name": "cf-container-logger",
-  "version": "0.0.7",
+  "version": "0.1.0",
   "description": "codefresh container logger",
   "keywords": [
     "cf-container-logger"
   ],
   "dependencies": {
-    "@codefresh-io/task-logger": "^0.0.18",
+    "@codefresh-io/task-logger": "1.5.3",
     "cf-errors": "^0.1.11",
     "cf-logs": "^1.1.0",
     "docker-events": "0.0.2",

service.yaml

@@ -1 +1 @@
-version: 1.0.36
+version: 1.0.37

yarn.lock

@@ -2,22 +2,24 @@
 # yarn lockfile v1
 
 
-"@codefresh-io/task-logger@^0.0.18":
-  version "0.0.18"
-  resolved "https://registry.yarnpkg.com/@codefresh-io/task-logger/-/task-logger-0.0.18.tgz#d7d69d4eeeaf9fc4d7ca78abf3a208a951d5f297"
-  integrity sha512-qDFO88Lkco5zHKGxV4YTNgHwfX5vgODleTgXyHuj1xdCuuZTsOvYmnm/qZcPVkiSb6tKSqKv3QBvMePKi6Ej8g==
+"@codefresh-io/task-logger@1.5.3":
+  version "1.5.3"
+  resolved "https://registry.yarnpkg.com/@codefresh-io/task-logger/-/task-logger-1.5.3.tgz#9465bd9cd98e63757ce5975dbf17e4341dd5066c"
+  integrity sha512-r/fL3QXZcYjIt2rabu/dZ6Ooeg1xCVHaNiUyjigEDiWOqPXOKpFh9FiTvqSNSC5QsK4Rs/bU6e8CU/thF9FyDg==
   dependencies:
     cf-errors "^0.1.11"
     crypto "0.0.3"
     debug "^4.1.1"
     firebase "git+https://github.com/codefresh-io/firebase.git#80b2ed883ff281cd67b53bd0f6a0bbd6f330fed5"
+    firebase-token-generator "^2.0.0"
     jsonwebtoken "^8.4.0"
     lodash "^3.10.1"
     mongodb "^3.1.13"
     node-redis-pubsub "^4.0.0"
     q "^1.4.1"
     redis "^2.8.0"
     request "^2.88.0"
+    requestretry "^4.0.2"
     retry "^0.12.0"
 
 "@sinonjs/commons@^1.0.2", "@sinonjs/commons@^1.3.0":
@@ -1230,7 +1232,7 @@ extend@^3.0.0, extend@~3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444"
 
-extend@~3.0.2:
+extend@^3.0.2, extend@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa"
   integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==
@@ -1365,6 +1367,11 @@ fined@^1.0.1:
     object.pick "^1.2.0"
     parse-filepath "^1.0.1"
 
+firebase-token-generator@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/firebase-token-generator/-/firebase-token-generator-2.0.0.tgz#9767d759ec13abdc99ba115fd5ea99d8b93d1206"
+  integrity sha1-l2fXWewTq9yZuhFf1eqZ2Lk9EgY=
+
 "firebase@git+https://github.com/codefresh-io/firebase.git#80b2ed883ff281cd67b53bd0f6a0bbd6f330fed5":
   version "2.4.2"
   resolved "git+https://github.com/codefresh-io/firebase.git#80b2ed883ff281cd67b53bd0f6a0bbd6f330fed5"
@@ -2791,6 +2798,11 @@ lodash@^4.0.0, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.4:
   version "4.17.4"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
 
+lodash@^4.17.10:
+  version "4.17.15"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548"
+  integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==
+
 lodash@^4.17.11:
   version "4.17.11"
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
@@ -3906,6 +3918,15 @@ request@^2.88.0:
     tunnel-agent "^0.6.0"
     uuid "^3.3.2"
 
+requestretry@^4.0.2:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/requestretry/-/requestretry-4.1.0.tgz#7e8a2247c7d58726d3efb5d25bb1915bca8f0be3"
+  integrity sha512-q3IT2vz5vkcMT6xgwB/BWzsmnu7N/27l9fW86U48gt9Mwrce5rSEyFvpAW7Il1/B78/NBUlYBvcCY1RzWUWy7w==
+  dependencies:
+    extend "^3.0.2"
+    lodash "^4.17.10"
+    when "^3.7.7"
+
 require_optional@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/require_optional/-/require_optional-1.0.1.tgz#4cf35a4247f64ca3df8c2ef208cc494b1ca8fc2e"
@@ -4735,6 +4756,11 @@ websocket-extensions@>=0.1.1:
   resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29"
   integrity sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==
 
+when@^3.7.7:
+  version "3.7.8"
+  resolved "https://registry.yarnpkg.com/when/-/when-3.7.8.tgz#c7130b6a7ea04693e842cdc9e7a1f2aa39a39f82"
+  integrity sha1-xxMLan6gRpPoQs3J56Hyqjmjn4I=
+
 which@^1.0.9, which@^1.1.1:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a"