commit
d7a1e75735
13 changed files with 2860 additions and 0 deletions
@ -0,0 +1 @@ |
|||||
|
node_modules/ |
||||
@ -0,0 +1,59 @@ |
|||||
|
--- |
||||
|
|
||||
|
kind: pipeline |
||||
|
type: docker |
||||
|
name: Waypoint |
||||
|
|
||||
|
environment: |
||||
|
WAYPOINT_SERVER_ADDR: waypoint.smaug.dev:9701 |
||||
|
WAYPOINT_SERVER_TLS: 1 |
||||
|
WAYPOINT_SERVER_TLS_SKIP_VERIFY: 1 |
||||
|
|
||||
|
steps: |
||||
|
- name: build |
||||
|
image: registry.gitlab.com/gitlab-org/waypoint-images:latest |
||||
|
volumes: |
||||
|
- name: dockersock |
||||
|
path: /var/run/docker.sock |
||||
|
commands: |
||||
|
- waypoint init |
||||
|
- waypoint build |
||||
|
environment: |
||||
|
WAYPOINT_SERVER_TOKEN: |
||||
|
from_secret: WAYPOINT_SERVER_TOKEN |
||||
|
|
||||
|
- name: deploy |
||||
|
image: registry.gitlab.com/gitlab-org/waypoint-images:latest |
||||
|
volumes: |
||||
|
- name: dockersock |
||||
|
path: /var/run/docker.sock |
||||
|
commands: |
||||
|
- waypoint deploy -local=false -release=false |
||||
|
environment: |
||||
|
WAYPOINT_SERVER_TOKEN: |
||||
|
from_secret: WAYPOINT_SERVER_TOKEN |
||||
|
|
||||
|
- name: release |
||||
|
image: registry.gitlab.com/gitlab-org/waypoint-images:latest |
||||
|
volumes: |
||||
|
- name: dockersock |
||||
|
path: /var/run/docker.sock |
||||
|
commands: |
||||
|
- waypoint release -local=false -prune-retain=0 |
||||
|
environment: |
||||
|
WAYPOINT_SERVER_TOKEN: |
||||
|
from_secret: WAYPOINT_SERVER_TOKEN |
||||
|
|
||||
|
volumes: |
||||
|
- name: dockersock |
||||
|
host: |
||||
|
path: /var/run/docker.sock |
||||
|
|
||||
|
trigger: |
||||
|
branch: |
||||
|
- master |
||||
|
- main |
||||
|
event: |
||||
|
- push |
||||
|
- pull_request |
||||
|
- custom |
||||
@ -0,0 +1,4 @@ |
|||||
|
.idea/ |
||||
|
node_modules/ |
||||
|
static/ |
||||
|
.env |
||||
@ -0,0 +1,43 @@ |
|||||
|
FROM node:16-slim |
||||
|
RUN apt-get update && apt-get install --no-install-recommends -yq \ |
||||
|
libgconf-2-4 libxss1 libxtst6 ca-certificates wget curl gnupg2 python \ |
||||
|
gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 \ |
||||
|
libexpat1 libfontconfig1 libgcc1 libgdk-pixbuf2.0-0 libglib2.0-0 \ |
||||
|
libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 \ |
||||
|
libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 \ |
||||
|
libxfixes3 libxi6 libxrandr2 libxrender1 \ |
||||
|
fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils \ |
||||
|
libatk-bridge2.0-0 ffmpeg |
||||
|
|
||||
|
ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE 1 |
||||
|
RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub |\ |
||||
|
apt-key add - |
||||
|
RUN echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list |
||||
|
RUN apt-get update && apt-get install -y google-chrome-unstable git \ |
||||
|
fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst --no-install-recommends \ |
||||
|
&& rm -rf /var/lib/apt/lists/* /src/*.deb |
||||
|
RUN mkdir -p /usr/share/fonts/emoji \ |
||||
|
&& curl --location --silent --show-error --out \ |
||||
|
/usr/share/fonts/emoji/emojione-android.ttf \ |
||||
|
https://github.com/emojione/emojione-assets/releases/download/3.1.2/emojione-android.ttf \ |
||||
|
&& chmod -R +rx /usr/share/fonts/ \ |
||||
|
&& fc-cache -fv |
||||
|
RUN rm /bin/sh && ln -s /bin/bash /bin/sh |
||||
|
|
||||
|
RUN mkdir /app |
||||
|
COPY package.json yarn.lock app/ |
||||
|
COPY src app/src |
||||
|
COPY config app/config |
||||
|
WORKDIR /app |
||||
|
RUN yarn |
||||
|
|
||||
|
WORKDIR /app |
||||
|
|
||||
|
COPY static ./static |
||||
|
|
||||
|
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true |
||||
|
RUN mkdir -p /home/node/Downloads \ |
||||
|
&& chown -R node:node /home/node \ |
||||
|
&& chown -R node:node /app |
||||
|
USER node |
||||
|
CMD yarn start |
||||
@ -0,0 +1,21 @@ |
|||||
|
MIT License |
||||
|
|
||||
|
Copyright (c) 2021 Logan Koester <logan@logankoester.com> |
||||
|
|
||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy |
||||
|
of this software and associated documentation files (the "Software"), to deal |
||||
|
in the Software without restriction, including without limitation the rights |
||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
||||
|
copies of the Software, and to permit persons to whom the Software is |
||||
|
furnished to do so, subject to the following conditions: |
||||
|
|
||||
|
The above copyright notice and this permission notice shall be included in all |
||||
|
copies or substantial portions of the Software. |
||||
|
|
||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
||||
|
SOFTWARE. |
||||
@ -0,0 +1,3 @@ |
|||||
|
# Page Recorder |
||||
|
|
||||
|
> A service for recording videos of a webpage |
||||
@ -0,0 +1,12 @@ |
|||||
|
export default { |
||||
|
viewport: { |
||||
|
width: 1280, |
||||
|
height: 720, |
||||
|
}, |
||||
|
fps: 60, |
||||
|
start: 0, |
||||
|
duration: 5, |
||||
|
quiet: true, |
||||
|
output: 'static/output.webm', |
||||
|
launchArguments: ['--disable-setuid-sandbox', '--no-sandbox'], |
||||
|
}; |
||||
@ -0,0 +1,32 @@ |
|||||
|
export default { |
||||
|
viewport: { |
||||
|
width: 1280, |
||||
|
height: 720, |
||||
|
}, |
||||
|
fps: 60, |
||||
|
start: 1, |
||||
|
duration: 5, |
||||
|
quiet: true, |
||||
|
output: 'static/output.webm', |
||||
|
launchArguments: ['--disable-setuid-sandbox', '--no-sandbox'], |
||||
|
selector: 'canvas', |
||||
|
preparePage: async function(page) { |
||||
|
await page.evaluate(async () => { |
||||
|
document.getElementById('clicktoplaydiv').hidden = true; |
||||
|
function playWhenReady() { |
||||
|
setTimeout(function () { |
||||
|
if (window.gtk && window.gtk.play) { |
||||
|
if (window.gtk.module.clickedToPlay) { |
||||
|
return false; |
||||
|
} else { |
||||
|
window.gtk.play(); |
||||
|
} |
||||
|
} else { |
||||
|
playWhenReady(); |
||||
|
} |
||||
|
}, 1000); |
||||
|
} |
||||
|
await playWhenReady(); |
||||
|
}); |
||||
|
}, |
||||
|
}; |
||||
@ -0,0 +1,7 @@ |
|||||
|
import defaultOptions from './default.js' |
||||
|
import dragonrubyOptions from './dragonruby.js' |
||||
|
|
||||
|
export default { |
||||
|
default: defaultOptions, |
||||
|
dragonruby: dragonrubyOptions, |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
{ |
||||
|
"name": "page-recorder", |
||||
|
"version": "1.0.0", |
||||
|
"description": "A service for recording videos of a webpage", |
||||
|
"main": "src/index.js", |
||||
|
"scripts": { |
||||
|
"start": "node src/index.js", |
||||
|
"dev": "nodemon src/index.js" |
||||
|
}, |
||||
|
"author": { |
||||
|
"name": "Logan Koester", |
||||
|
"url": "https://ldk.dev", |
||||
|
"email": "logan@logankoester.com" |
||||
|
}, |
||||
|
"license": "MIT", |
||||
|
"type": "module", |
||||
|
"dependencies": { |
||||
|
"@hapi/catbox-redis": "^6.0.2", |
||||
|
"@hapi/hapi": "^20.2.1", |
||||
|
"@hapi/inert": "^6.0.4", |
||||
|
"@hapi/vision": "^6.1.0", |
||||
|
"filenamify-url": "^3.0.0", |
||||
|
"hapi-auth-jwt2": "^10.2.0", |
||||
|
"hapi-dev-errors": "^4.0.0", |
||||
|
"hapi-pino": "^9.1.0", |
||||
|
"hapi-swagger": "^14.2.4", |
||||
|
"moment": "^2.29.1", |
||||
|
"pino-pretty": "^7.3.0", |
||||
|
"puppeteer": "^10.4.0", |
||||
|
"timecut": "^0.3.1" |
||||
|
}, |
||||
|
"devDependencies": { |
||||
|
"@types/node": "^16.11.1", |
||||
|
"nodemon": "^2.0.15" |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,274 @@ |
|||||
|
import Hapi from '@hapi/hapi'; |
||||
|
import Boom from '@hapi/boom'; |
||||
|
import inert from '@hapi/inert'; |
||||
|
import CatboxRedis from '@hapi/catbox-redis'; |
||||
|
import vision from '@hapi/vision'; |
||||
|
import devErrors from 'hapi-dev-errors'; |
||||
|
import hapiPino from 'hapi-pino'; |
||||
|
import HapiSwagger from 'hapi-swagger'; |
||||
|
import HapiAuthJwt2 from 'hapi-auth-jwt2'; |
||||
|
import jwt from 'jsonwebtoken'; |
||||
|
import filenamifyUrl from 'filenamify-url'; |
||||
|
import moment from 'moment'; |
||||
|
import Joi from 'joi'; |
||||
|
import timecut from 'timecut'; |
||||
|
import path from 'path'; |
||||
|
import { readFile } from 'fs/promises'; |
||||
|
import environments from '../config/index.js'; |
||||
|
|
||||
|
const Package = JSON.parse(await readFile(new URL('../package.json', import.meta.url))); |
||||
|
|
||||
|
const defaultOptions = (environment = 'default') => ( |
||||
|
environments[environment] |
||||
|
); |
||||
|
|
||||
|
const init = async () => { |
||||
|
|
||||
|
const server = Hapi.server({ |
||||
|
port: process.env.PORT || 8080, |
||||
|
host: process.env.HOST || 'localhost', |
||||
|
debug: false, |
||||
|
cache: [ |
||||
|
{ |
||||
|
name: 'capture_service_cache', |
||||
|
provider: { |
||||
|
constructor: CatboxRedis, |
||||
|
options: { |
||||
|
//partition: 'my_cached_data',
|
||||
|
host: (process.env.REDIS_HOST || 'localhost'), |
||||
|
port: (process.env.REDIS_PORT || 6379), |
||||
|
database: 0, |
||||
|
//tls: {},
|
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
], |
||||
|
}); |
||||
|
|
||||
|
const swaggerOptions = { |
||||
|
info: { |
||||
|
title: Package.name, |
||||
|
description: Package.description, |
||||
|
version: Package.version, |
||||
|
contact: Package.author, |
||||
|
}, |
||||
|
schemes: ['http', 'https'], |
||||
|
host: `${process.env.HOST || 'localhost'}:${process.env.PORT || 8080}`, |
||||
|
basePath: '/', |
||||
|
debug: false, |
||||
|
documentationPath: '/', |
||||
|
expanded: 'full', |
||||
|
tryItOutEnabled: false, |
||||
|
securityDefinitions: { |
||||
|
jwt: { |
||||
|
type: 'apiKey', |
||||
|
name: 'Authorization', |
||||
|
in: 'header', |
||||
|
}, |
||||
|
}, |
||||
|
security: [{ jwt: [] }], |
||||
|
}; |
||||
|
|
||||
|
const apiClients = { |
||||
|
'default': { |
||||
|
id: 'default', |
||||
|
secret: 'letmein', |
||||
|
} |
||||
|
}; |
||||
|
|
||||
|
const privateKey = (process.env.PRIVATE_KEY || '5oauN6YdPF'); |
||||
|
|
||||
|
const validate = decoded => { |
||||
|
if (!apiClients[decoded.id]) { |
||||
|
return { isValid: false }; |
||||
|
} else { |
||||
|
return { isValid: true }; |
||||
|
} |
||||
|
}; |
||||
|
|
||||
|
await server.register(inert); |
||||
|
await server.register({ |
||||
|
plugin: devErrors, |
||||
|
options: { |
||||
|
showErrors: process.env.NODE_ENV !== 'production' |
||||
|
} |
||||
|
}); |
||||
|
await server.register({ |
||||
|
plugin: hapiPino, |
||||
|
options: { |
||||
|
logQueryParams: false, |
||||
|
logRequestComplete: false, |
||||
|
redact: ['req.headers.authorization'], |
||||
|
getChildBindings: (request) => ({ }), |
||||
|
transport: { |
||||
|
target: 'pino-pretty', |
||||
|
options: { |
||||
|
colorize: true, |
||||
|
minimumLevel: 'info', |
||||
|
levelFirst: true, |
||||
|
messageFormat: true, |
||||
|
timestampKey: 'time', |
||||
|
translateTime: true, |
||||
|
singleLine: false, |
||||
|
mkdir: true, |
||||
|
append: true, |
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
}); |
||||
|
await server.register([ |
||||
|
HapiAuthJwt2, |
||||
|
inert, |
||||
|
vision, |
||||
|
{ |
||||
|
plugin: HapiSwagger, |
||||
|
options: swaggerOptions, |
||||
|
}, |
||||
|
]); |
||||
|
|
||||
|
server.auth.strategy('jwt', 'jwt', { |
||||
|
key: privateKey, |
||||
|
validate, |
||||
|
verifyOptions: { algorithms: ['HS256'] } |
||||
|
}); |
||||
|
|
||||
|
server.auth.default('jwt'); |
||||
|
|
||||
|
const createVideo = async (opts={}) => { |
||||
|
const environment = opts['environment'] || 'default'; |
||||
|
const options = Object.assign(defaultOptions(environment), opts); |
||||
|
server.logger.info('Recording video for %s', options.url); |
||||
|
try { |
||||
|
await timecut(options); |
||||
|
server.logger.info('Video written to %s', options.output); |
||||
|
} catch (e) { |
||||
|
throw Error(e); |
||||
|
} |
||||
|
return options.output; |
||||
|
}; |
||||
|
|
||||
|
server.method('video.create', createVideo, { |
||||
|
cache: { |
||||
|
cache: 'capture_service_cache', |
||||
|
staleIn: 60 * 60 * 1000, |
||||
|
staleTimeout: 1, |
||||
|
expiresIn: 24 * 60 * 60 * 1000, |
||||
|
generateTimeout: 5 * 60 * 1000, |
||||
|
pendingGenerateTimeout: 60 * 1000, |
||||
|
getDecoratedValue: true, |
||||
|
}, |
||||
|
generateKey: (options) => options.url, |
||||
|
}); |
||||
|
|
||||
|
server.route({ |
||||
|
method: 'GET', |
||||
|
path: '/video', |
||||
|
handler: async (request, h) => { |
||||
|
const { url, fps, start, duration, selector, environment } = request.query; |
||||
|
const output = path.join('./', 'static', `${filenamifyUrl(url)}.webm`); |
||||
|
const { cached, value, report } = await server.methods.video.create({ |
||||
|
url, |
||||
|
output, |
||||
|
fps, |
||||
|
start, |
||||
|
duration, |
||||
|
selector, |
||||
|
environment, |
||||
|
}); |
||||
|
|
||||
|
if (cached) { |
||||
|
request.logger.info( |
||||
|
'Cache is %s for %s (expiring %s)', |
||||
|
(cached.isStale ? 'stale' : 'fresh'), |
||||
|
cached.item, |
||||
|
moment.duration(cached.ttl).humanize(true, { h: 48 }), |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
return h.file(value).header('Content-Disposition', 'inline; filename=video.webm'); |
||||
|
}, |
||||
|
options: { |
||||
|
tags: ['api'], |
||||
|
auth: false, // THIS DISABLES JWT AUTH!
|
||||
|
description: 'Fetch a video recording', |
||||
|
plugins: { |
||||
|
'hapi-swagger': { |
||||
|
produces: ['video/webm'], |
||||
|
responses: { |
||||
|
200: { |
||||
|
description: 'A WebM video file', |
||||
|
schema: Joi.binary().meta({ swaggerType: 'file' }), |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
}, |
||||
|
validate: { |
||||
|
query: Joi.object({ |
||||
|
url: Joi.string().uri().required().description('The URL of the target page'), |
||||
|
selector: Joi.string().description('Crop each frame to the bounding box of the first item found by the specified CSS selector'), |
||||
|
fps: Joi.number().integer().min(1).max(60).default(60).description('Framerate to record as video'), |
||||
|
start: Joi.number().integer().min(0).default(0).description('Seconds to wait before saving any frames'), |
||||
|
duration: Joi.number().integer().min(1).default(5).description('Duration to capture (in seconds)'), |
||||
|
environment: Joi.string().default('default').description('Choose a set of default options from a file at config/*.js'), |
||||
|
}), |
||||
|
}, |
||||
|
}, |
||||
|
}); |
||||
|
|
||||
|
server.route({ |
||||
|
method: 'GET', |
||||
|
path: '/token/{id}/{secret}', |
||||
|
options: { |
||||
|
auth: false, |
||||
|
tags: ['api'], |
||||
|
description: 'Request a token for a given API key', |
||||
|
handler: (request, h) => { |
||||
|
const { id, secret } = request.params; |
||||
|
if (!apiClients[id]) { |
||||
|
throw Boom.badRequest(`API client "${id}" does not exist`); |
||||
|
} |
||||
|
if (apiClients[id].secret != secret) { |
||||
|
throw Boom.unauthorized(`Secret does not match the expected value`); |
||||
|
} |
||||
|
const token = jwt.sign({ id: id }, privateKey, { algorithm: 'HS256' }); |
||||
|
return { token: token }; |
||||
|
}, |
||||
|
validate: { |
||||
|
params: Joi.object({ |
||||
|
id: Joi.string().required().description('A registered API client'), |
||||
|
secret: Joi.string().required().description('A matching client secret'), |
||||
|
}), |
||||
|
}, |
||||
|
}, |
||||
|
}); |
||||
|
|
||||
|
server.ext('onPreResponse', (request, h) => { |
||||
|
const { url } = request.query; |
||||
|
const response = request.response; |
||||
|
if (response.isBoom && |
||||
|
response.output.statusCode === 404) { |
||||
|
request.logger.debug(request); |
||||
|
if (request.route.method === 'get' && |
||||
|
request.route.path === '/video') { |
||||
|
request.logger.warn( |
||||
|
'Dropping cache key %s (%s)', |
||||
|
url, |
||||
|
response.output.payload.error |
||||
|
); |
||||
|
server.methods.video.create.cache.drop({ url }); |
||||
|
return h.response().redirect(request.url); |
||||
|
} |
||||
|
} |
||||
|
return h.continue; |
||||
|
}); |
||||
|
|
||||
|
await server.start(); |
||||
|
server.logger.info('Server running on %s', server.info.uri); |
||||
|
}; |
||||
|
|
||||
|
process.on('unhandledRejection', (err) => { |
||||
|
console.log(err); |
||||
|
process.exit(1); |
||||
|
}); |
||||
|
|
||||
|
init(); |
||||
@ -0,0 +1,75 @@ |
|||||
|
project = "page-recorder" |
||||
|
|
||||
|
runner { |
||||
|
enabled = true |
||||
|
data_source "git" { |
||||
|
url = "git@git.smaug.dev:ereborstudios/page-recorder.git" |
||||
|
ref = "master" |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
# Labels can be specified for organizational purposes. |
||||
|
# labels = { "foo" = "bar" } |
||||
|
|
||||
|
# An application to deploy. |
||||
|
app "page-recorder" { |
||||
|
|
||||
|
config { |
||||
|
env = { |
||||
|
REDIS_HOST = var.redis_host |
||||
|
REDIS_PORT = var.redis_port |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
labels = { |
||||
|
"service" = var.name |
||||
|
} |
||||
|
|
||||
|
# Build specifies how an application should be deployed. In this case, |
||||
|
# we'll build using a Dockerfile and keeping it in a local registry. |
||||
|
build { |
||||
|
use "docker" { |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
# Deploy to Docker |
||||
|
deploy { |
||||
|
use "docker" { |
||||
|
service_port = 3090 |
||||
|
static_environment = { |
||||
|
"NODE_ENV": "production" |
||||
|
} |
||||
|
|
||||
|
labels = { |
||||
|
"traefik.http.routers.${var.name}.rule" = "Host(`${var.host}`, `${var.name}.waypoint.smaug.dev`)" |
||||
|
"traefik.http.routers.${var.name}.entrypoints" = "websecure" |
||||
|
"traefik.http.routers.${var.name}.tls.certresolver" = "myresolver" |
||||
|
"traefik.http.services.${var.name}.loadbalancer.server.port" = "3090" |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
variable "name" { |
||||
|
type = string |
||||
|
default = "page-recorder" |
||||
|
description = "Project name" |
||||
|
} |
||||
|
|
||||
|
variable "host" { |
||||
|
type = string |
||||
|
default = "preview.smaug.dev" |
||||
|
description = "Project host" |
||||
|
} |
||||
|
|
||||
|
variable "redis_host" { |
||||
|
type = string |
||||
|
default = "redis.smaug.dev" |
||||
|
description = "Redis host" |
||||
|
} |
||||
|
|
||||
|
variable "redis_port" { |
||||
|
type = string |
||||
|
default = 6379 |
||||
|
description = "Redis port" |
||||
|
} |
||||
File diff suppressed because it is too large
Loading…
Reference in new issue