Merge pull request #1 from bigo-frontend/open-source

feat：开源版本

.autod.conf.js

@@ -0,0 +1,28 @@
+'use strict';
+
+module.exports = {
+  write: true,
+  plugin: 'autod-egg',
+  prefix: '^',
+  devprefix: '^',
+  exclude: [
+    'test/fixtures',
+    'coverage',
+  ],
+  dep: [
+    'egg',
+    'egg-scripts',
+  ],
+  devdep: [
+    'autod',
+    'autod-egg',
+    'egg-bin',
+    'tslib',
+    'typescript',
+  ],
+  keep: [
+  ],
+  semver: [
+  ],
+  test: 'scripts',
+};

.eslintignore

@@ -0,0 +1,2 @@
+**/*.d.ts
+node_modules/

.eslintrc

@@ -0,0 +1,10 @@
+{
+  "extends": "eslint-config-egg/typescript",
+  "rules": {
+    "@typescript-eslint/ban-ts-ignore": 1,
+    "@typescript-eslint/no-var-requires": 0
+  },
+  "parserOptions": {
+    "project": "./tsconfig.json"
+  }
+}

.gitignore

@@ -0,0 +1,27 @@
+logs/
+npm-debug.log
+node_modules/
+coverage/
+.idea/
+run/
+.DS_Store
+.vscode
+*.swp
+*.lock
+package-lock.json
+.travis.yml
+appveyor.yml
+
+app/**/*.js
+test/**/*.js
+!test/mockData/**/*.js
+config/**/*.js
+app/**/*.map
+test/**/*.map
+config/**/*.map
+typings/app/**/*.d.ts
+typings/config/**/*.d.ts
+projects
+projects_build
+temp
+app/public

LICENSE

@@ -1,6 +1,6 @@
-MIT License
+The MIT License (MIT)
 
-Copyright (c) 2021 bigo前端
+Copyright (c) 2021-present, bigo
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9,13 +9,13 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE

README.md

@@ -1,2 +1,26 @@
-# morning-paper-sever
-前端早报服务端
+## 前端早报
+
+## 系统介绍
+bigo前端早报系统，包括但不限于掘金、知乎、infoq等咨询站点内容推送
+支持推送到：企业微信
+待支持：钉钉
+
+## 开发
+
+```bash
+$ npm i
+$ npm run dev
+$ open http://localhost:9001/
+```
+
+## 功能预览
+![](https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/cad986da6a884001b9a9e5e333061a83~tplv-k3u1fbpfcp-watermark.image)
+
+![](https://p1-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/c6f7ffe7c5fd4bff817d009166d3d786~tplv-k3u1fbpfcp-watermark.image)
+
+![](https://p3-juejin.byteimg.com/tos-cn-i-k3u1fbpfcp/a3ee7778d27a46da9f46f93289ac0901~tplv-k3u1fbpfcp-watermark.image)
+
+## 实现原理
+
+使用puppeteer获取html结构，然后使用cheerio解析dom结构，返回爬虫数据。
+

app/common/index.ts

@@ -0,0 +1,17 @@
+// 爬虫站点枚举
+export enum EBizType {
+  INFOQ = 'infoq',
+  JUEJIN = 'juejin',
+  SEGMENTFAULT = 'segmentfault',
+  ZHIHU = 'zhihu',
+  LEETCODE = 'leetcode',
+  TESTERHOME = 'testerhome',
+  TENCENTTMQ = 'tencenttmq',
+  MEITUAN = 'meituan',
+  YOUZAN = 'youzan',
+  GITHUBISSUES = 'githubIssues',
+  FREECODECAMP = 'freecodecamp',
+  MEDIUM = 'medium',
+  DEVTO = 'devto',
+  CSSTRICKS = 'csstricks',
+}

app/common/scanFolder.ts

@@ -0,0 +1,49 @@
+const fs = require('fs');
+
+/**
+ * 遍历文件夹，返回下面的文件以及目录
+ *
+ * @param {String} pathFolder 目标目录or目标文件
+ * @param {RegExp} ignoreList 忽略哪些文件夹
+ * @param {RegExp} includeList 包含哪些文件
+ */
+export default function(pathFolder: string, ignoreList?, includeList?) {
+  const fileList: string[] = [];
+  const folderList: string[] = [];
+  function walk(pathFolder) {
+    if (ignoreList && pathFolder.match(ignoreList)) { // 忽略哪些文件夹
+      return true;
+    }
+    if (fs.statSync(pathFolder).isFile()) { // 如果是目标文件，直接返回
+      fileList.push(pathFolder);
+      return true;
+    }
+    folderList.push(pathFolder);
+    const files = fs.readdirSync(pathFolder);
+    files.forEach(function(item) {
+      const tmpPath = pathFolder + '/' + item,
+        stats = fs.statSync(tmpPath);
+      if (stats.isDirectory()) {
+        walk(tmpPath);
+        folderList.push(tmpPath);
+      } else {
+        if (ignoreList && tmpPath.match(ignoreList)) { // 忽略哪些文件
+          return true;
+        }
+        if (includeList) { // 包含哪些文件
+          if (tmpPath.match(includeList)) {
+            fileList.push(tmpPath);
+          }
+          return true;
+        }
+        fileList.push(tmpPath);
+      }
+    });
+  }
+
+  walk(pathFolder);
+  return {
+    fileList,
+    folderList,
+  };
+}

app/controller/baseController.ts

@@ -0,0 +1,27 @@
+import { Controller } from 'egg';
+
+export default class BaseController extends Controller {
+  /**
+   * 成功
+   * @param data 响应数据
+   */
+  success(data) {
+    this.ctx.body = {
+      success: true,
+      ...data,
+    };
+    return;
+  }
+
+  /**
+   * 失败
+   * @param data 响应数据
+   */
+  fail(data) {
+    this.ctx.body = {
+      success: false,
+      ...data,
+    };
+    return;
+  }
+}

app/controller/home.ts

@@ -0,0 +1,9 @@
+import Controller from '@/controller/baseController';
+
+export default class HomeController extends Controller {
+  public async index() {
+    this.ctx.body = 'hello world';
+    return;
+  }
+
+}

app/controller/morningPaper.ts

@@ -0,0 +1,90 @@
+import Controller from '@/controller/baseController';
+import { EBizType } from '@/common';
+
+export default class MorningPaper extends Controller {
+  /**
+   * 早报爬虫
+   * infoq ==> http://127.0.0.1:9001/morningPaper?link=https://www.infoq.cn/topic/Front-end&bizType=infoq&waitTime=3000
+   * juejin ==> http://127.0.0.1:9001/morningPaper?link=https://juejin.cn/frontend&bizType=juejin&waitTime=3000
+   * segmentfault ==> http://127.0.0.1:9001/morningPaper?link=https://segmentfault.com/channel/frontend&bizType=segmentfault&waitTime=3000
+   * zhihu ==> http://127.0.0.1:9001/morningPaper?link=https://zhuanlan.zhihu.com/eggjs&bizType=zhihu&waitTime=3000
+   * leetcode ==> http://127.0.0.1:9001/morningPaper?link=https://leetcode-cn.com/problems/course-schedule&bizType=leetcode&waitTime=3000
+   * testerhome ==> http://127.0.0.1:9001/morningPaper?link=https://testerhome.com/topics/last&bizType=testerhome&waitTime=3000
+   * tencenttmq ==> http://127.0.0.1:9001/morningPaper?link=https://cloud.tencent.com/developer/column/1088&bizType=tencenttmq&waitTime=3000
+   * githubIssues ==> http://127.0.0.1:9001/morningPaper?link=https://github.com/bigo-frontend/blog/issues&bizType=githubIssues&waitTime=3000
+   * freecodecamp ==> http://127.0.0.1:9001/morningPaper?link=https://www.freecodecamp.org/news/&bizType=freecodecamp&waitTime=3000
+   * medium ==> http://127.0.0.1:9001/morningPaper?link=https://medium.com/front-end-weekly&bizType=medium&waitTime=3000
+   * devto ==> http://127.0.0.1:9001/morningPaper?link=https://dev.to&bizType=devto&waitTime=3000
+   * csstricks ==> http://127.0.0.1:9001/morningPaper?link=https://css-tricks.com/&bizType=csstricks&waitTime=3000
+   */
+  public async index() {
+    const waitTime = Number(this.ctx.query.waitTime);
+    let link = this.ctx.query.link;
+    const bizType = this.ctx.query.bizType;
+    let html = '';
+    if (!link) {
+      this.fail({
+        msg: '入参校验不通过',
+      });
+      return;
+    }
+    if (bizType === EBizType.LEETCODE) { // leetcode先获取每日一题链接，再爬取具体内容
+      link = await this.service.spider.leetcode.index.getRandomOneQuestionLink();
+      if (link === '') {
+        this.fail({
+          msg: '入参校验不通过',
+        });
+        return;
+      }
+    }
+    console.log(link);
+    const htmlResult = await this.service.puppeteer.index.getHtml(link, waitTime);
+    if (htmlResult.status === false) {
+      this.fail({
+        msg: '爬取html失败，请稍后重试或者调整超时时间',
+      });
+      return;
+    }
+    html = htmlResult.data;
+    const links = this.service.morningPaper.index.formatHtmlByBizType(bizType, html) || [];
+    this.success({
+      data: links.filter(item => !item.title.match('招聘')),
+    });
+    return;
+  }
+
+  /**
+   * 推送消息到企业微信机器人
+   * http://127.0.0.1:9001/sendMsg2Weixin?content=hello,world!&token=企业微信token
+   */
+  async sendMsg2Weixin() {
+    const content = this.ctx.query.content;
+    const token = this.ctx.query.token;
+    if (!token || !content) {
+      this.fail({
+        resultObj: {
+          msg: '入参数据异常',
+        },
+      });
+      return;
+    }
+    const status = await this.service.sendMsg.weixin.index(token, content);
+    this.logger.info('发送消息 bizType=%j  content=%j', token, content);
+    if (status) {
+      this.success({
+        resultObj: {
+          msg: '发送成功',
+        },
+      });
+      return;
+    }
+
+    this.fail({
+      resultObj: {
+        msg: '发送失败',
+      },
+    });
+    return;
+  }
+
+}

app/middleware/catchError.ts

@@ -0,0 +1,16 @@
+module.exports = () => {
+  return async (ctx, next) => {
+
+    try {
+      await next();
+    } catch (err) {
+      ctx.logger.error(err);
+      ctx.body = {
+        status: false,
+        msg: '出错啦~',
+      };
+    }
+
+    return ctx.body;
+  };
+};

app/middleware/httpLogs.ts

@@ -0,0 +1,16 @@
+module.exports = () => {
+  return async (ctx, next) => {
+
+    await next();
+
+    ctx.app.logger.info('httplogs=%j', {
+      body: ctx.body,
+      url: ctx.request.url,
+      method: ctx.request.method,
+      referer: ctx.request.headers.referer,
+      resStatus: ctx.response.status,
+    });
+
+    return ctx.body;
+  };
+};

app/router.ts

@@ -0,0 +1,11 @@
+import { Application } from 'egg';
+
+export default (app: Application) => {
+  const { controller, router } = app;
+
+  router.get('/', controller.home.index);
+  // 早报咨询获取
+  router.get('/morningPaper', controller.morningPaper.index);
+  // 推送微信机器人消息
+  router.get('/sendMsg2Weixin', controller.morningPaper.sendMsg2Weixin);
+};