From d555d58917531f79ca4396a0a81e26a50d9caef6 Mon Sep 17 00:00:00 2001 From: Clement Bois Date: Mon, 17 Jun 2019 17:06:01 +0200 Subject: [PATCH] v0.0.1 --- .env.sample | 4 ++- README.md | 41 +++++++++++++++++++---------- package.json | 3 ++- src/App.ts | 41 ++++++++++++++++++++--------- src/Rest.ts | 59 +++++++++++++++++++++++++++++++----------- src/actions.json | 6 +++++ src/database.json | 15 +++++++++++ src/prepare.ts | 7 +++-- src/tools/data/push.ts | 9 ++++--- src/tools/token/get.ts | 38 +++++++++++++++++++++++++++ src/types/config.ts | 2 ++ src/utils/Limiter.ts | 52 +++++++++++++++++++++++++++++++++++++ tsconfig.json | 1 + 13 files changed, 230 insertions(+), 48 deletions(-) create mode 100644 src/actions.json create mode 100644 src/database.json create mode 100644 src/tools/token/get.ts create mode 100644 src/utils/Limiter.ts diff --git a/.env.sample b/.env.sample index 604ae75..690c345 100644 --- a/.env.sample +++ b/.env.sample @@ -1,4 +1,6 @@ DOMAIN=mastodon.social TOKEN=xxxxxxxxxxxxxxxx ROOT_STATUS=yyyyyyyyyy -TIMEOUT=60000 \ No newline at end of file +TIMEOUT=60000 +LIMIT_COUNT=150 +LIMIT_TIME=300000 \ No newline at end of file diff --git a/README.md b/README.md index 9135214..c74d702 100644 --- a/README.md +++ b/README.md @@ -10,27 +10,24 @@ A pretty simple random message bot using [Mastodon](https://joinmastodon.org) st ### Mastodon - Create an account for this bot. Please mark it as bot in configuration page. -- In settings page, create a new application with read write permission and copy the private key. -- Create a status to store actions (could be private) and copy its id. *Called `root status` after* +- In settings page, create a new application with `read write` permission and copy the private key. + - Or `read:accounts read:statuses write:statuses write:media` + - Could use `npm run token-add` *(requires `DOMAIN` in `.env`)* +- Create a status to store actions (could be private) containing `#botodon` and copy its id. *Called `root status` after* - Create a status to store content (could be private) and copy its id. *Called `data status` after* - Reply to this status to add content. (Removing @acct, to limit self spam) - Reply to `root status` with options tags *(See `action status`)* like `#botodon #global #data_from_{DATA_STATUS_ID}` ### Botodon -```sh -git clone botodon -cd botodon -npm install -cp .env.sample .env -``` +Download folder from [releases](https://git.wadza.fr/me/botodon/releases). In `.env` set - `DOMAIN={INSTANCE_DOMAIN}` -- `TOKEN={ACCOUNT_PRIVATE_KEY}` +- `TOKEN={PRIVATE_KEY}` - `ROOT_STATUS={ROOT_STATUS_ID}` -Run `npm run start` with cron. +Run `node index.js` with cron. Enjoy @@ -50,11 +47,12 @@ There is 3 *types* of statuses: Its a folder containing `action statuses` and global options as tags. tag | description --- | -- +--- | --- botodon | **Required** Enable tags processing async | Enable actions parallel processing deep | Use replies to actions as actions shared | **Unsafe** Include other users actions +file | Also load actions from `actions.json` ### Action status @@ -71,13 +69,14 @@ Send standard status with random content from `xxxxxxxxxxxx` status #### Options tag | default | description --- | -- | -- +--- | --- | --- botodon | false | **Required** Enable tags processing **Data source** | | content send +data_file | empty | Use `database.json` as source *(See database.sample.json)* data_from_**ID** | empty | Add a source of content: `data status` id data_deep | false | Use replies to content as content data_shared | false | **Warning** Include other users content -data_tagged_**TAG** | empty | Require content to have this tag +data_tagged_**TAG** | empty | Require content to have this tag *(Tags are lowercase on backend)* data_favourited | false | Require content to be fav by bot account data_favourites_**N** | false | Require at least N favourites data_last_**N** | disabled | Only include N last statuses @@ -90,7 +89,7 @@ followers_of_**ID** | empty | Send to each followers of given account id **Don't replies_to_**ID** | empty | Send to each repliers of given status id replies_deep | false | Include replies to replies replies_visibility | false | Use reply visibility -favourites_**ID** | empty | Send ti each use who fav given status id +favourites_**ID** | empty | Send to each use who fav given status id visibility | unlisted | Visibility to use. One of `public`, `unlisted`, `private`, `direct` @@ -102,8 +101,20 @@ Add `.env.XXX` files and run `BOT_NAME=XXX node index.js` Mastodon context API use an arbitrary limit of `4096` since [#7564](https://github.com/tootsuite/mastodon/pull/7564), it's so the limit were new replies are excluded. +Mastodon api limit to 300 requests within 5 minutes. Using a rate limiter at 150 requests for 5 minutes. + Followers soft limit of `999` *(must include pagination)* and hard limit of `7500`. +### Pleroma + +It supports Mastodon API but uses case-sensitive ids which can't be stored as tag. + +You have to use actions and database json files as fallback. + +## Tools + +`tools` folder contains some useful script which could be as helpful as dangerous. + ## TODO - Add abstract and inherit on deep actions @@ -111,3 +122,5 @@ Followers soft limit of `999` *(must include pagination)* and hard limit of `750 - Use followers pagination - Add options on `data status` - Add move pick options +- Handle errors +- Move to Go ? diff --git a/package.json b/package.json index 4b3d841..35c7b1b 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "action-add": "ts-node src/tools/action/add.ts", "action-clear": "ts-node src/tools/action/clear.ts", "action-list": "ts-node src/tools/action/list.ts", - "data-push": "ts-node src/tools/data/push.ts" + "data-push": "ts-node src/tools/data/push.ts", + "token-get": "ts-node src/tools/token/get.ts" }, "keywords": [ "mastodon", diff --git a/src/App.ts b/src/App.ts index 31ca9c6..32be04f 100644 --- a/src/App.ts +++ b/src/App.ts @@ -1,4 +1,6 @@ -import { VisibilityType } from 'mastodon' +import { Status, VisibilityType } from 'mastodon' +import actionSource from './actions.json' +import dataSource from './database.json' import Rest from './Rest' import { Action, ActionConfig, ActionConfigData, RootConfig, VISIBILITIES } from './types/config' import Logger from './utils/Logger' @@ -7,6 +9,14 @@ import Selector from './utils/Selector' export default class App { + static filterData(replies: Status[], config: ActionConfigData) { + return replies + .filter(s => !config.favourited || s.favourited) + .filter(s => s.favourites_count >= config.favourites) + .filter(s => config.tagged.every(tag => s.tags.map(t => t.name).includes(tag))) + .slice(0, config.last === -1 ? undefined : config.last) + } + constructor(private rest: Rest) { } async run(rootId: string) { @@ -23,7 +33,7 @@ export default class App { return } - for (const action of await this.loadActions(rootId, config.deep, config.shared ? undefined : me)) { + for (const action of await this.loadActions(rootId, config.deep, config.file, config.shared ? undefined : me)) { const promise = process(me, action) if (!config.async) { await promise @@ -50,13 +60,14 @@ export default class App { async: false, botodon: false, deep: false, - shared: false + shared: false, + file: false }, status.tags.map(t => t.name).reverse()) } - async loadActions(id: string, deep: boolean, account?: string) { + async loadActions(id: string, deep: boolean, file: boolean, account?: string) { const lines: Action[] = (await this.rest.getReplies(id, deep, account)) - .map(s => ({ id: s.id, tags: s.tags.map(t => t.name) })) + .map(s => ({ id: s.id, tags: s.tags.map(t => t.name) })).concat(file ? actionSource : []) Logger.debug(`Found ${lines.length} action(s)`) if (!lines.length) { @@ -70,6 +81,7 @@ export default class App { const config = matcher({ botodon: false, data: { + file: false, from: [], deep: false, shared: false, @@ -104,7 +116,7 @@ export default class App { // Data const datas = (await Promise.all(config.data.from.map(id => this.loadData(id, config.data, me)))) - .reduce((a, b) => a.concat(b)) + .reduce((a, b) => a.concat(b), config.data.file ? this.loadDataFile(config.data) : []) if (!datas.length) { Logger.error(`Action ${action.id}: Any content`) return @@ -121,6 +133,7 @@ export default class App { config.followers_of.push(me) } for await (const followers of config.followers_of.map(id => this.rest.getFollowers(id))) { + Logger.debug(followers) targets.push(...followers.map(({ acct }) => ({ acct, visibility: config.visibility }))) } for await (const replies of config.replies.to.map(id => this.rest.getReplies(id, config.replies.deep))) { @@ -150,13 +163,17 @@ export default class App { } async loadData(id: string, config: ActionConfigData, me: string) { - const replies = await this.rest.getReplies(id, config.deep, config.shared ? undefined : me) + return App.filterData(await this.rest.getReplies(id, config.deep, config.shared ? undefined : me), config) + } - return replies - .filter(s => !config.favourited || s.favourited) - .filter(s => s.favourites_count >= config.favourites) - .filter(s => config.tagged.every(tag => s.tags.map(t => t.name).includes(tag))) - .slice(0, config.last === -1 ? undefined : config.last) + loadDataFile(config: ActionConfigData) { + return App.filterData(dataSource.filter(s => config.deep || !s.deep).map(s => ({ + id: 'local', uri: 'file', account: { id: 'local', acct: 'file', bot: true, display_name: 'file', emojis: [] }, content: s.content || '', + created_at: '', emojis: [], favourited: s.favourited || false, favourites_count: 0, media_attachments: (s.medias || []).map(m => ({ + id: m, description: '', url: 'file', preview_url: 'file', type: 'img' + })), sensitive: false, reblogged: false, reblogs_count: 0, + replies_count: 0, visibility: 'direct', tags: (s.tags || []).map(t => ({ name: t })) + })), config) } } \ No newline at end of file diff --git a/src/Rest.ts b/src/Rest.ts index 455387f..34faa20 100644 --- a/src/Rest.ts +++ b/src/Rest.ts @@ -1,19 +1,21 @@ -import { RequestPromiseAPI } from 'request-promise-native' +import { RequestPromise, RequestPromiseAPI } from 'request-promise-native' import { Account, Context, Status, StatusPost } from './types/mastodon' +import Limiter from './utils/Limiter' +import Logger from './utils/Logger'; export default class Rest { - constructor(readonly api: RequestPromiseAPI) {} + constructor(private api: RequestPromiseAPI, private limiter: Limiter) {} - async getMe(): Promise { - return this.api.get('accounts/verify_credentials') + async getMe() { + return this.get('accounts/verify_credentials') } - async getStatus(id: string): Promise { - return this.api.get(`statuses/${id}`) + async getStatus(id: string) { + return this.get(`statuses/${id}`) } - async getContext(id: string): Promise { - return this.api.get(`statuses/${id}/context`) + async getContext(id: string) { + return this.get(`statuses/${id}/context`) } async getDescendants(id: string) { @@ -27,22 +29,49 @@ export default class Rest { .filter(s => !account || s.account.id === account)) } - async getFollowers(id: string): Promise { + async getFollowers(id: string) { // TODO: use link header - return this.api.get(`account/${id}/followers`, { qs: { limit: 999 } }) + return this.get(`accounts/${id}/followers`, { limit: 999 }) } - async getFavouritedBy(id: string): Promise { + async getFavouritedBy(id: string) { // TODO: use link header - return this.api.get(`statuses/${id}/favourited_by`, { qs: { limit: 999 } }) + return this.get(`statuses/${id}/favourited_by`, { limit: 999 }) } - async postStatus(status: StatusPost): Promise { - return this.api.post('statuses', { body: status }) + async postStatus(status: StatusPost) { + return this.post('statuses', status) } async deleteStatus(id: string) { - return this.api.delete(`statuses/${id}`) + return this.delete(`statuses/${id}`) + } + + async postApp(redirectUri: string, scopes: string) { + return this.post<{ client_id: string, client_secret: string }>('/apps', { + client_name: 'botodon', + redirect_uris: redirectUri, scopes, + website: 'https://git.wadza.fr/me/botodon' + }) + } + + protected async call(promise: RequestPromise) { + return this.limiter.promise(promise.catch(err => { + Logger.error(`Rest: ${err.message} on ${err.options.uri}`) + throw err + }) as undefined as Promise) + } + + protected async get(url: string, qs: object = {}) { + return this.call(this.api.get(url, { qs })) + } + + protected async post(url: string, body: any) { + return this.call(this.api.post(url, { body })) + } + + protected async delete(url: string) { + return this.call(this.api.delete(url)) } } \ No newline at end of file diff --git a/src/actions.json b/src/actions.json new file mode 100644 index 0000000..4a4c1ea --- /dev/null +++ b/src/actions.json @@ -0,0 +1,6 @@ +[ + { + "id": "local-sample", + "tags": ["botodon", "data_file"] + } +] \ No newline at end of file diff --git a/src/database.json b/src/database.json new file mode 100644 index 0000000..64d75cb --- /dev/null +++ b/src/database.json @@ -0,0 +1,15 @@ +[ + { + "content": "[EN] Text\n\n[FR] Texte", + "weight": 42, + "favourited": true + }, + { + "content": "Texte", + "tags": ["fr"] + }, + { + "medias": ["id"], + "deep": true + } +] \ No newline at end of file diff --git a/src/prepare.ts b/src/prepare.ts index c4c4d8e..8847183 100644 --- a/src/prepare.ts +++ b/src/prepare.ts @@ -3,15 +3,18 @@ import path from 'path' import rp from 'request-promise-native' import Rest from './Rest' +import Limiter from './utils/Limiter'; dotenv.config({ path: path.resolve(process.cwd(), process.env.BOT_NAME ? `.env.${process.env.BOT_NAME}` : '.env') }) +const toInt = (s: string) => Number.parseInt(s, undefined) + export const rest = new Rest(rp.defaults({ auth: { bearer: process.env.TOKEN }, baseUrl: `https://${process.env.DOMAIN}/api/v1/`, - timeout: Number.parseInt(process.env.TIMEOUT, undefined), + timeout: toInt(process.env.TIMEOUT), json: true -})) +}), new Limiter(toInt(process.env.LIMIT_COUNT), toInt(process.env.LIMIT_TIME))) export const rootStatus = process.env.ROOT_STATUS \ No newline at end of file diff --git a/src/tools/data/push.ts b/src/tools/data/push.ts index e20c1af..2656002 100644 --- a/src/tools/data/push.ts +++ b/src/tools/data/push.ts @@ -9,13 +9,16 @@ async function run() { return } - const post = async (in_reply_to_id: string, status: string) => rest.postStatus({ - in_reply_to_id, status, + const post = async (inReplyToId: string, status: string) => rest.postStatus({ + in_reply_to_id: inReplyToId, + status, visibility: visibility as any || 'direct' }).then(s => s.id) + // TODO: push database.json const data = JSON.parse(fs.readFileSync(file, 'utf8')) const folder = await post(rootStatus, title || file) + Logger.info('Content length', data.length) for (const row of data) { if (typeof row === 'string') { await post(folder, row) @@ -29,6 +32,6 @@ async function run() { throw new Error('bad type ' + typeof row) } } - Logger.info('data added', folder) + Logger.info('Data added', folder) } run() \ No newline at end of file diff --git a/src/tools/token/get.ts b/src/tools/token/get.ts new file mode 100644 index 0000000..ffa8631 --- /dev/null +++ b/src/tools/token/get.ts @@ -0,0 +1,38 @@ +import querystring from 'querystring' +import readline from 'readline' +import rp from 'request-promise-native' +import { rest } from '../../prepare' +import Logger from '../../utils/Logger' + +async function run() { + try { + Logger.info('Checking current token') + const { acct: me } = await rest.getMe() + Logger.info(`Already logged as ${me}`) + } catch (error) { + Logger.info('Trying to get token') + const redirectUri = 'urn:ietf:wg:oauth:2.0:oob' + const scopes = 'read:accounts read:statuses write:statuses write:media' + + const { client_id, client_secret } = await rest.postApp(redirectUri, scopes) + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }) + Logger.info(`Please visit: https://${process.env.DOMAIN}/oauth/authorize?scope=${querystring.escape(scopes)}&redirect_uri=${redirectUri}&response_type=code&client_id=${client_id} to validate your token`) + rl.question('And then paste token here: ', async code => { + const { access_token } = await rp.post(`https://${process.env.DOMAIN}/oauth/token`, { + json: true, + body: { + client_id, client_secret, + grant_type: 'authorization_code', + code, redirect_uri: redirectUri + } + }) + rl.write(`Put TOKEN=${access_token} in .env`) + rl.close() + }) + } +} +run() \ No newline at end of file diff --git a/src/types/config.ts b/src/types/config.ts index 0d3c98d..be01c7f 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -5,6 +5,7 @@ export interface RootConfig { async: boolean deep: boolean shared: boolean + file: boolean } export interface Action { id: string, @@ -27,6 +28,7 @@ export interface ActionConfig { } export interface ActionConfigData { + file: boolean from: string[] deep: boolean shared: boolean diff --git a/src/utils/Limiter.ts b/src/utils/Limiter.ts new file mode 100644 index 0000000..87d10cf --- /dev/null +++ b/src/utils/Limiter.ts @@ -0,0 +1,52 @@ +import Logger from "./Logger"; + +export default class Limiter { + + private past: number[] = [] + private queue: any[] = [] + private timer: NodeJS.Timeout | NodeJS.Immediate = null + + constructor(private calls: number, private time: number, private sync: boolean = false) { } + + dequeue() { + const now = Date.now() + + this.past = this.past.filter(t => now - t < this.time) + + while (this.past.length < this.calls && this.queue.length > 0) { + this.past.push(now) + const action = this.queue.shift() + action() + + if (this.sync) { + break + } + } + + if (this.queue.length <= 0) { + this.timer = null + } else { + const delay = this.sync ? this.time / this.calls : this.time - now + this.past[0] + Logger.warn(`Limiter: waiting ${delay}ms...`) + this.timer = setTimeout(this.dequeue.bind(this), delay) + } + } + + limit(callback: CallableFunction) { + this.queue.push(callback) + if (this.timer === null) { + this.timer = setImmediate(this.dequeue.bind(this)) + } + } + + promise(promise: Promise) { + let res: (value?: T) => void = null + let rej: (reason?: any) => void = null + const wrapper = new Promise((resolve, reject) => { + res = resolve + rej = reject + }) + this.limit(() => promise.then(res).catch(rej)) + return wrapper + } +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index d0392f9..9f46d88 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,7 @@ "target": "es6", "noImplicitAny": true, "moduleResolution": "node", + "resolveJsonModule": true, "sourceMap": false, "outDir": "dist", "baseUrl": ".",