Skip to content

Commit dbf1deb

Browse files
vinitshahdeovinitshahdeo
authored
Merge pull request #9 from priyanshu-kun/feature/tests
[Feature/tests] Introduce controller tests, implement validation service, and perform refactoring of utility files.
2 parents 2ca871c + 3f097c0 commit dbf1deb

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+37434
-0
lines changed

‎src/server/.editorconfig

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
################################################
2+
# ╔═╗╔╦╗╦╔╦╗╔═╗╦═╗┌─┐┌─┐┌┐┌┌─┐┬┌─┐
3+
# ║╣ ║║║ ║ ║ ║╠╦╝│ │ ││││├┤ ││ ┬
4+
# o╚═╝═╩╝╩ ╩ ╚═╝╩╚═└─┘└─┘┘└┘└ ┴└─┘
5+
#
6+
# > Formatting conventions for your Sails app.
7+
#
8+
# This file (`.editorconfig`) exists to help
9+
# maintain consistent formatting throughout the
10+
# files in your Sails app.
11+
#
12+
# For the sake of convention, the Sails team's
13+
# preferred settings are included here out of the
14+
# box. You can also change this file to fit your
15+
# team's preferences (for example, if all of the
16+
# developers on your team have a strong preference
17+
# for tabs over spaces),
18+
#
19+
# To review what each of these options mean, see:
20+
# http://editorconfig.org/
21+
#
22+
################################################
23+
root = true
24+
25+
[*]
26+
indent_style = space
27+
indent_size = 2
28+
end_of_line = lf
29+
charset = utf-8
30+
trim_trailing_whitespace = true
31+
insert_final_newline = true

‎src/server/.eslintignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎src/server/.eslintrc

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{
2+
// ╔═╗╔═╗╦ ╦╔╗╔╔╦╗┬─┐┌─┐
3+
// ║╣ ╚═╗║ ║║║║ ║ ├┬┘│
4+
// o╚═╝╚═╝╩═╝╩╝╚╝ ╩ ┴└─└─┘
5+
// A set of basic code conventions designed to encourage quality and consistency
6+
// across your Sails app's code base. These rules are checked against
7+
// automatically any time you run `npm test`.
8+
//
9+
// > Note: If you're using mocha, you'll want to add an extra override file to your
10+
// > `test/` folder so that eslint will tolerate mocha-specific globals like `before`
11+
// > and `describe`.
12+
// Designed for ESLint v4.
13+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
14+
// For more information about any of the rules below, check out the relevant
15+
// reference page on eslint.org. For example, to get details on "no-sequences",
16+
// you would visit `http://eslint.org/docs/rules/no-sequences`. If you're unsure
17+
// or could use some advice, come by https://sailsjs.com/support.
18+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
19+
20+
"env": {
21+
"node": true
22+
},
23+
24+
"parserOptions": {
25+
"ecmaVersion": 2018
26+
},
27+
28+
"globals": {
29+
// If "no-undef" is enabled below, be sure to list all global variables that
30+
// are used in this app's backend code (including the globalIds of models):
31+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
32+
"Promise": true,
33+
"sails": true,
34+
"_": true
35+
// …and any others (e.g. `"Organization": true`)
36+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
37+
},
38+
39+
"rules": {
40+
"block-scoped-var": ["error"],
41+
"callback-return": ["error", ["done", "proceed", "next", "onwards", "callback", "cb"]],
42+
"camelcase": ["warn", {"properties":"always"}],
43+
"comma-style": ["warn", "last"],
44+
"curly": ["warn"],
45+
"eqeqeq": ["error", "always"],
46+
"eol-last": ["warn"],
47+
"handle-callback-err": ["error"],
48+
"indent": ["warn", 2, {
49+
"SwitchCase": 1,
50+
"MemberExpression": "off",
51+
"FunctionDeclaration": {"body":1, "parameters":"off"},
52+
"FunctionExpression": {"body":1, "parameters":"off"},
53+
"CallExpression": {"arguments":"off"},
54+
"ArrayExpression": 1,
55+
"ObjectExpression": 1,
56+
"ignoredNodes": ["ConditionalExpression"]
57+
}],
58+
"linebreak-style": ["error", "unix"],
59+
"no-dupe-keys": ["error"],
60+
"no-duplicate-case": ["error"],
61+
"no-extra-semi": ["warn"],
62+
"no-labels": ["error"],
63+
"no-mixed-spaces-and-tabs": [2, "smart-tabs"],
64+
"no-redeclare": ["warn"],
65+
"no-return-assign": ["error", "always"],
66+
"no-sequences": ["error"],
67+
"no-trailing-spaces": ["warn"],
68+
"no-undef": ["off"],
69+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
70+
// ^^Note: If this "no-undef" rule is enabled (set to `["error"]`), then all model globals
71+
// (e.g. `"Organization": true`) should be included above under "globals".
72+
// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
73+
"no-unexpected-multiline": ["warn"],
74+
"no-unreachable": ["warn"],
75+
"no-unused-vars": ["warn", {"caughtErrors":"all", "caughtErrorsIgnorePattern": "^unused($|[A-Z].*$)", "argsIgnorePattern": "^unused($|[A-Z].*$)", "varsIgnorePattern": "^unused($|[A-Z].*$)" }],
76+
"no-use-before-define": ["error", {"functions":false}],
77+
"one-var": ["warn", "never"],
78+
"prefer-arrow-callback": ["warn", {"allowNamedFunctions":true}],
79+
"quotes": ["warn", "single", {"avoidEscape":false, "allowTemplateLiterals":true}],
80+
"semi": ["warn", "always"],
81+
"semi-spacing": ["warn", {"before":false, "after":true}],
82+
"semi-style": ["warn", "last"]
83+
}
84+
85+
}

‎src/server/.gitignore

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
################################################
2+
# ┌─┐┬┌┬┐╦╔═╗╔╗╔╔═╗╦═╗╔═╗
3+
# │ ┬│ │ ║║ ╦║║║║ ║╠╦╝║╣
4+
# o└─┘┴ ┴ ╩╚═╝╝╚╝╚═╝╩╚═╚═╝
5+
#
6+
# > Files to exclude from your app's repo.
7+
#
8+
# This file (`.gitignore`) is only relevant if
9+
# you are using git.
10+
#
11+
# It exists to signify to git that certain files
12+
# and/or directories should be ignored for the
13+
# purposes of version control.
14+
#
15+
# This keeps tmp files and sensitive credentials
16+
# from being uploaded to your repository. And
17+
# it allows you to configure your app for your
18+
# machine without accidentally committing settings
19+
# which will smash the local settings of other
20+
# developers on your team.
21+
#
22+
# Some reasonable defaults are included below,
23+
# but, of course, you should modify/extend/prune
24+
# to fit your needs!
25+
#
26+
################################################
27+
28+
29+
################################################
30+
# Local Configuration
31+
#
32+
# Explicitly ignore files which contain:
33+
#
34+
# 1. Sensitive information you'd rather not push to
35+
# your git repository.
36+
# e.g., your personal API keys or passwords.
37+
#
38+
# 2. Developer-specific configuration
39+
# Basically, anything that would be annoying
40+
# to have to change every time you do a
41+
# `git pull` on your laptop.
42+
# e.g. your local development database, or
43+
# the S3 bucket you're using for file uploads
44+
# during development.
45+
#
46+
################################################
47+
48+
config/local.js
49+
50+
51+
################################################
52+
# Dependencies
53+
#
54+
#
55+
# When releasing a production app, you _could_
56+
# hypothetically include your node_modules folder
57+
# in your git repo, but during development, it
58+
# is always best to exclude it, since different
59+
# developers may be working on different kernels,
60+
# where dependencies would need to be recompiled
61+
# anyway.
62+
#
63+
# Most of the time, the node_modules folder can
64+
# be excluded from your code repository, even
65+
# in production, thanks to features like the
66+
# package-lock.json file / NPM shrinkwrap.
67+
#
68+
# But no matter what, since this is a Sails app,
69+
# you should always push up the package-lock.json
70+
# or shrinkwrap file to your repository, to avoid
71+
# accidentally pulling in upgraded dependencies
72+
# and breaking your code.
73+
#
74+
# That said, if you are having trouble with
75+
# dependencies, (particularly when using
76+
# `npm link`) this can be pretty discouraging.
77+
# But rather than just adding the lockfile to
78+
# your .gitignore, try this first:
79+
# ```
80+
# rm -rf node_modules
81+
# rm package-lock.json
82+
# npm install
83+
# ```
84+
#
85+
# [?] For more tips/advice, come by and say hi
86+
# over at https://sailsjs.com/support
87+
#
88+
################################################
89+
90+
node_modules
91+
92+
93+
################################################
94+
#
95+
# > Do you use bower?
96+
# > re: the bower_components dir, see this:
97+
# > http://addyosmani.com/blog/checking-in-front-end-dependencies/
98+
# > (credit Addy Osmani, @addyosmani)
99+
#
100+
################################################
101+
102+
103+
################################################
104+
# Temporary files generated by Sails/Waterline.
105+
################################################
106+
107+
.tmp
108+
109+
110+
################################################
111+
# Miscellaneous
112+
#
113+
# Common files generated by text editors,
114+
# operating systems, file systems, dbs, etc.
115+
################################################
116+
117+
*~
118+
*#
119+
.DS_STORE
120+
.netbeans
121+
nbproject
122+
.idea
123+
*.iml
124+
.vscode
125+
.node_history
126+
dump.rdb
127+
128+
npm-debug.log
129+
lib-cov
130+
*.seed
131+
*.log
132+
*.out
133+
*.pid
134+
135+
dist

‎src/server/.npmrc

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
######################
2+
# ╔╗╔╔═╗╔╦╗┬─┐┌─┐ #
3+
# ║║║╠═╝║║║├┬┘│ #
4+
# o╝╚╝╩ ╩ ╩┴└─└─┘ #
5+
######################
6+
7+
# Hide NPM log output unless it is related to an error of some kind:
8+
loglevel=error
9+
10+
# Make "npm audit" an opt-in thing for subsequent installs within this app:
11+
audit=false

‎src/server/.sailsrc

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"generators": {
3+
"modules": {}
4+
},
5+
"_generatedWith": {
6+
"sails": "1.5.4",
7+
"sails-generate": "2.0.8"
8+
},
9+
"hooks": {
10+
"grunt": false
11+
}
12+
}

‎src/server/README.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# server
2+
3+
a [Sails v1](https://sailsjs.com) application
4+
5+
6+
### Links
7+
8+
+ [Sails framework documentation](https://sailsjs.com/get-started)
9+
+ [Version notes / upgrading](https://sailsjs.com/documentation/upgrading)
10+
+ [Deployment tips](https://sailsjs.com/documentation/concepts/deployment)
11+
+ [Community support options](https://sailsjs.com/support)
12+
+ [Professional / enterprise options](https://sailsjs.com/enterprise)
13+
14+
15+
### Version info
16+
17+
This app was originally generated on Sat Jun 10 2023 12:22:10 GMT+0530 (India Standard Time) using Sails v1.5.4.
18+
19+
<!-- Internally, Sails used [`sails-generate@2.0.8`](https://github.com/balderdashy/sails-generate/tree/v2.0.8/lib/core-generators/new). -->
20+
21+
22+
23+
<!--
24+
Note: Generators are usually run using the globally-installed `sails` CLI (command-line interface). This CLI version is _environment-specific_ rather than app-specific, thus over time, as a project's dependencies are upgraded or the project is worked on by different developers on different computers using different versions of Node.js, the Sails dependency in its package.json file may differ from the globally-installed Sails CLI release it was originally generated with. (Be sure to always check out the relevant [upgrading guides](https://sailsjs.com/upgrading) before upgrading the version of Sails used by your app. If you're stuck, [get help here](https://sailsjs.com/support).)
25+
-->
26+

‎src/server/api/constants/Constants.js

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
const CC_SERVER_URL = 'https://index.commoncrawl.org/';
2+
const RABBIT_MQ_URL = 'amqp://localhost:5672';
3+
const QUEUE_NAME = 'index-files-jobs';
4+
const FILE_PATH = `${__dirname}/../../dist/data/output.txt`;
5+
const RESULTS_FILE_PATH = `${__dirname}/../../dist/results/results.txt`;
6+
const DIST_PATH = `${__dirname}/../../dist`;
7+
const CHUNK_SIZE = 1024 * 1024 * 2;
8+
const PROGRESS_BAR_WIDTH = 50; // Width of the progress bar
9+
const URL_REGEX = /http[s]?:\/\/(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+/gi;
10+
const API_DEFINITION_REGEX = /^(openapi|swagger)\.(json|yaml|yml)(\?[\w=&]+)?$/;
11+
const SUB_DOMAIN_REGEX = /^(?:https?:\/\/)?api\.[^\/\s]+\.[^\/\s]+(?:\/[^\/\s]*)*$/gi;
12+
const KEYWORD_REGEX = /^(?:https?:\/\/)?[^\/\s]+(?:\/[^\/\s]+)*(?:\/(openapi|swagger))(?:\/[^\/\s]*)*$/gi;
13+
const BASE_URL = 'http://localhost:1337';
14+
15+
module.exports = { CC_SERVER_URL, RABBIT_MQ_URL, QUEUE_NAME, FILE_PATH, CHUNK_SIZE, PROGRESS_BAR_WIDTH, URL_REGEX, API_DEFINITION_REGEX, SUB_DOMAIN_REGEX, KEYWORD_REGEX, RESULTS_FILE_PATH, DIST_PATH, BASE_URL };

‎src/server/api/controllers/CrawlingController.js

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
const { cleanDB } = require('../utils/DBUtils');
2+
const { saveIndexFiles } = require('../utils/DBUtils');
3+
const { selectDataSources } = require('../utils/SelectDataSourcesUtils');
4+
5+
module.exports = {
6+
/**
7+
* Endpoint for starting the crawling process.
8+
* Controller for the 'start-crawling' route. Initiates the crawling process based on the provided data source.
9+
* Retrieves API definitions from the selected data source and sends the response accordingly.
10+
*
11+
* @route POST /api/v1/run/crawler
12+
* @param {Object} req - The request object.
13+
* @param {Object} res - The response object.
14+
* @param {Object} req.body - The payload of the request.
15+
* @param {string} req.body.dataSource - The data source for crawling.
16+
* @param {boolean} req.query.latest - Indicates whether to use the latest data.
17+
* @returns {Array<string>} An array of API definition URLs, or an error response.
18+
* @throws {Error} If an error occurs during the crawling process.
19+
*
20+
* @example
21+
* Request body example:
22+
* {
23+
* 'dataSource': 'commonCrawl'
24+
* }
25+
*
26+
* @example
27+
* Query parameter example:
28+
* /api/v1/run/crawler?latest=true
29+
* or
30+
* /api/v1/run/crawler?latest=false
31+
*
32+
* @example
33+
* Response example:
34+
* [
35+
* 'https://api.example.com/definition1',
36+
* 'https://api.example.com/definition2',
37+
* ...
38+
* ]
39+
*/
40+
startCrawling: async function (req, res) {
41+
try {
42+
const { dataSource } = req.body;
43+
/**
44+
* If req.query.latest is truthy and equals the string 'true', then latest is set to true; else latest is set to false;
45+
* otherwise, it is set to the default value true.
46+
*/
47+
const latest = req.query.latest ? req.query.latest === 'true' : true;
48+
49+
if (!dataSource) {
50+
return res.badRequest('Data source not provided');
51+
}
52+
53+
const indexFiles = await selectDataSources(dataSource, latest);
54+
55+
if(indexFiles.length === 0) {
56+
return res.notFound('Cannot able to scrape index files.');
57+
}
58+
59+
await cleanDB(IndexFilesModel);
60+
await saveIndexFiles(indexFiles);
61+
62+
return res.json({ indexFiles });
63+
} catch (error) {
64+
return res.serverError(error);
65+
}
66+
},
67+
};

0 commit comments

Comments
 (0)