v0.0.1

.env.sample

@@ -1,4 +1,6 @@
 DOMAIN=mastodon.social
 TOKEN=xxxxxxxxxxxxxxxx
 ROOT_STATUS=yyyyyyyyyy
-TIMEOUT=60000
+TIMEOUT=60000
+LIMIT_COUNT=150
+LIMIT_TIME=300000

README.md

@@ -10,27 +10,24 @@ A pretty simple random message bot using [Mastodon](https://joinmastodon.org) st
 ### Mastodon
 
 - Create an account for this bot. Please mark it as bot in configuration page.
-- In settings page, create a new application with read write permission and copy the private key.
+- In settings page, create a new application with `read write` permission and copy the private key.
-- Create a status to store actions (could be private) and copy its id. *Called `root status` after*
+  - Or `read:accounts read:statuses write:statuses write:media`
+  - Could use `npm run token-add` *(requires `DOMAIN` in `.env`)*
+- Create a status to store actions (could be private) containing `#botodon` and copy its id. *Called `root status` after*
 - Create a status to store content (could be private) and copy its id. *Called `data status` after*
   - Reply to this status to add content. (Removing @acct, to limit self spam)
 - Reply to `root status` with options tags *(See `action status`)* like `#botodon #global #data_from_{DATA_STATUS_ID}`
 
 ### Botodon
 
-```sh
+Download folder from [releases](https://git.wadza.fr/me/botodon/releases).
-git clone <url> botodon
-cd botodon
-npm install
-cp .env.sample .env
-```
 
 In `.env` set
 - `DOMAIN={INSTANCE_DOMAIN}`
-- `TOKEN={ACCOUNT_PRIVATE_KEY}`
+- `TOKEN={PRIVATE_KEY}`
 - `ROOT_STATUS={ROOT_STATUS_ID}`
 
-Run `npm run start` with cron.
+Run `node index.js` with cron.
 
 Enjoy
 
@@ -50,11 +47,12 @@ There is 3 *types* of statuses:
 Its a folder containing `action statuses` and global options as tags.
 
 tag | description
--- | --
+--- | ---
 botodon | **Required** Enable tags processing
 async | Enable actions parallel processing
 deep | Use replies to actions as actions
 shared | **Unsafe** Include other users actions
+file | Also load actions from `actions.json` 
 
 ### Action status
 
@@ -71,13 +69,14 @@ Send standard status with random content from `xxxxxxxxxxxx` status
 #### Options
 
 tag | default | description
--- | -- | --
+--- | --- | ---
 botodon | false | **Required** Enable tags processing
 **Data source** | | content send
+data_file | empty | Use `database.json` as source *(See database.sample.json)*
 data_from_**ID** | empty | Add a source of content: `data status` id
 data_deep | false | Use replies to content as content
 data_shared | false | **Warning** Include other users content
-data_tagged_**TAG** | empty | Require content to have this tag
+data_tagged_**TAG** | empty | Require content to have this tag *(Tags are lowercase on backend)*
 data_favourited | false | Require content to be fav by bot account
 data_favourites_**N** | false | Require at least N favourites
 data_last_**N** | disabled | Only include  N last statuses
@@ -90,7 +89,7 @@ followers_of_**ID** | empty | Send to each followers of given account id **Don't
 replies_to_**ID** | empty | Send to each repliers of given status id
 replies_deep | false | Include replies to replies
 replies_visibility | false | Use reply visibility
-favourites_**ID** | empty | Send ti each use who fav given status id
+favourites_**ID** | empty | Send to each use who fav given status id
 visibility | unlisted | Visibility to use. One of `public`, `unlisted`, `private`, `direct`
 
 
@@ -102,8 +101,20 @@ Add `.env.XXX` files and run `BOT_NAME=XXX node index.js`
 
 Mastodon context API use an arbitrary limit of `4096` since [#7564](https://github.com/tootsuite/mastodon/pull/7564), it's so the limit were new replies are excluded.
 
+Mastodon api limit to 300 requests within 5 minutes. Using a rate limiter at 150 requests for 5 minutes.
+
 Followers soft limit of `999` *(must include pagination)* and hard limit of `7500`.
 
+### Pleroma
+
+It supports Mastodon API but uses case-sensitive ids which can't be stored as tag.
+
+You have to use actions and database json files as fallback.
+
+## Tools
+
+`tools` folder contains some useful script which could be as helpful as dangerous.
+
 ## TODO
 
 - Add abstract and inherit on deep actions
@@ -111,3 +122,5 @@ Followers soft limit of `999` *(must include pagination)* and hard limit of `750
 - Use followers pagination
 - Add options on `data status`
 - Add move pick options
+- Handle errors
+- Move to Go ?

package.json

@@ -12,7 +12,8 @@
     "action-add": "ts-node src/tools/action/add.ts",
     "action-clear": "ts-node src/tools/action/clear.ts",
     "action-list": "ts-node src/tools/action/list.ts",
-    "data-push": "ts-node src/tools/data/push.ts"
+    "data-push": "ts-node src/tools/data/push.ts",
+    "token-get": "ts-node src/tools/token/get.ts"
   },
   "keywords": [
     "mastodon",

src/App.ts

@@ -1,4 +1,6 @@
-import { VisibilityType } from 'mastodon'
+import { Status, VisibilityType } from 'mastodon'
+import actionSource from './actions.json'
+import dataSource from './database.json'
 import Rest from './Rest'
 import { Action, ActionConfig, ActionConfigData, RootConfig, VISIBILITIES } from './types/config'
 import Logger from './utils/Logger'
@@ -7,6 +9,14 @@ import Selector from './utils/Selector'
 
 export default class App {
 
+  static filterData(replies: Status[], config: ActionConfigData) {
+    return replies
+      .filter(s => !config.favourited || s.favourited)
+      .filter(s => s.favourites_count >= config.favourites)
+      .filter(s => config.tagged.every(tag => s.tags.map(t => t.name).includes(tag)))
+      .slice(0, config.last === -1 ? undefined : config.last)
+  }
+
   constructor(private rest: Rest) { }
 
   async run(rootId: string) {
@@ -23,7 +33,7 @@ export default class App {
       return
     }
 
-    for (const action of await this.loadActions(rootId, config.deep, config.shared ? undefined : me)) {
+    for (const action of await this.loadActions(rootId, config.deep, config.file, config.shared ? undefined : me)) {
       const promise = process(me, action)
       if (!config.async) {
         await promise
@@ -50,13 +60,14 @@ export default class App {
       async: false,
       botodon: false,
       deep: false,
-      shared: false
+      shared: false,
+      file: false
     }, status.tags.map(t => t.name).reverse())
   }
 
-  async loadActions(id: string, deep: boolean, account?: string) {
+  async loadActions(id: string, deep: boolean, file: boolean, account?: string) {
     const lines: Action[] = (await this.rest.getReplies(id, deep, account))
-      .map(s => ({ id: s.id, tags: s.tags.map(t => t.name) }))
+      .map(s => ({ id: s.id, tags: s.tags.map(t => t.name) })).concat(file ? actionSource : [])
 
     Logger.debug(`Found ${lines.length} action(s)`)
     if (!lines.length) {
@@ -70,6 +81,7 @@ export default class App {
     const config = matcher<ActionConfig>({
       botodon: false,
       data: {
+        file: false,
         from: [],
         deep: false,
         shared: false,
@@ -104,7 +116,7 @@ export default class App {
 
     // Data
     const datas = (await Promise.all(config.data.from.map(id => this.loadData(id, config.data, me))))
-      .reduce((a, b) => a.concat(b))
+      .reduce((a, b) => a.concat(b), config.data.file ? this.loadDataFile(config.data) : [])
     if (!datas.length) {
       Logger.error(`Action ${action.id}: Any content`)
       return
@@ -121,6 +133,7 @@ export default class App {
       config.followers_of.push(me)
     }
     for await (const followers of config.followers_of.map(id => this.rest.getFollowers(id))) {
+      Logger.debug(followers)
       targets.push(...followers.map(({ acct }) => ({ acct, visibility: config.visibility })))
     }
     for await (const replies of config.replies.to.map(id => this.rest.getReplies(id, config.replies.deep))) {
@@ -150,13 +163,17 @@ export default class App {
   }
 
   async loadData(id: string, config: ActionConfigData, me: string) {
-    const replies = await this.rest.getReplies(id, config.deep, config.shared ? undefined : me)
+    return App.filterData(await this.rest.getReplies(id, config.deep, config.shared ? undefined : me), config)
+  }
 
-    return replies
+  loadDataFile(config: ActionConfigData) {
-      .filter(s => !config.favourited || s.favourited)
+    return App.filterData(dataSource.filter(s => config.deep || !s.deep).map(s => ({
-      .filter(s => s.favourites_count >= config.favourites)
+      id: 'local', uri: 'file', account: { id: 'local', acct: 'file', bot: true, display_name: 'file', emojis: [] }, content: s.content || '',
-      .filter(s => config.tagged.every(tag => s.tags.map(t => t.name).includes(tag)))
+      created_at: '', emojis: [], favourited: s.favourited || false, favourites_count: 0, media_attachments: (s.medias || []).map(m => ({
-      .slice(0, config.last === -1 ? undefined : config.last)
+        id: m, description: '', url: 'file', preview_url: 'file', type: 'img'
+      })), sensitive: false, reblogged: false, reblogs_count: 0,
+      replies_count: 0, visibility: 'direct', tags: (s.tags || []).map(t => ({ name: t }))
+    })), config)
   }
 
 }

src/Rest.ts

@@ -1,19 +1,21 @@
-import { RequestPromiseAPI } from 'request-promise-native'
+import { RequestPromise, RequestPromiseAPI } from 'request-promise-native'
 import { Account, Context, Status, StatusPost } from './types/mastodon'
+import Limiter from './utils/Limiter'
+import Logger from './utils/Logger';
 
 export default class Rest {
-  constructor(readonly api: RequestPromiseAPI) {}
+  constructor(private api: RequestPromiseAPI, private limiter: Limiter) {}
 
-  async getMe(): Promise<Account> {
+  async getMe() {
-    return this.api.get('accounts/verify_credentials')
+    return this.get<Account>('accounts/verify_credentials')
   }
 
-  async getStatus(id: string): Promise<Status> {
+  async getStatus(id: string) {
-    return this.api.get(`statuses/${id}`)
+    return this.get<Status>(`statuses/${id}`)
   }
 
-  async getContext(id: string): Promise<Context> {
+  async getContext(id: string) {
-    return this.api.get(`statuses/${id}/context`)
+    return this.get<Context>(`statuses/${id}/context`)
   }
 
   async getDescendants(id: string) {
@@ -27,22 +29,49 @@ export default class Rest {
         .filter(s => !account || s.account.id === account))
   }
 
-  async getFollowers(id: string): Promise<Account[]> {
+  async getFollowers(id: string) {
     // TODO: use link header
-    return this.api.get(`account/${id}/followers`, { qs: { limit: 999 } })
+    return this.get<Account[]>(`accounts/${id}/followers`, { limit: 999 })
   }
 
-  async getFavouritedBy(id: string): Promise<Account[]> {
+  async getFavouritedBy(id: string) {
     // TODO: use link header
-    return this.api.get(`statuses/${id}/favourited_by`, { qs: { limit: 999 } })
+    return this.get<Account[]>(`statuses/${id}/favourited_by`, { limit: 999 })
   }
 
-  async postStatus(status: StatusPost): Promise<Status> {
+  async postStatus(status: StatusPost) {
-    return this.api.post('statuses', { body: status })
+    return this.post<Status>('statuses', status)
   }
 
   async deleteStatus(id: string) {
-    return this.api.delete(`statuses/${id}`)
+    return this.delete<void>(`statuses/${id}`)
+  }
+
+  async postApp(redirectUri: string, scopes: string) {
+    return this.post<{ client_id: string, client_secret: string }>('/apps', {
+      client_name: 'botodon',
+      redirect_uris: redirectUri, scopes,
+      website: 'https://git.wadza.fr/me/botodon'
+    })
+  }
+
+  protected async call<T>(promise: RequestPromise) {
+    return this.limiter.promise<T>(promise.catch(err => {
+      Logger.error(`Rest: ${err.message} on ${err.options.uri}`)
+      throw err
+    }) as undefined as Promise<T>)
+  }
+
+  protected async get<T>(url: string, qs: object = {}) {
+    return this.call<T>(this.api.get(url, { qs }))
+  }
+
+  protected async post<T>(url: string, body: any) {
+    return this.call<T>(this.api.post(url, { body }))
+  }
+
+  protected async delete<T>(url: string) {
+    return this.call<T>(this.api.delete(url))
   }
 
 }

src/actions.json

@@ -0,0 +1,6 @@
+[
+    {
+        "id": "local-sample",
+        "tags": ["botodon", "data_file"]
+    }
+]

src/database.json

@@ -0,0 +1,15 @@
+[
+    {
+        "content": "[EN] Text\n\n[FR] Texte",
+        "weight": 42,
+        "favourited": true
+    },
+    {
+        "content": "Texte",
+        "tags": ["fr"]
+    },
+    {
+        "medias": ["id"],
+        "deep": true
+    }
+]

src/prepare.ts

@@ -3,15 +3,18 @@ import path from 'path'
 import rp from 'request-promise-native'
 
 import Rest from './Rest'
+import Limiter from './utils/Limiter';
 
 dotenv.config({ path: path.resolve(process.cwd(), process.env.BOT_NAME ? `.env.${process.env.BOT_NAME}` : '.env') })
 
+const toInt = (s: string) => Number.parseInt(s, undefined)
+
 export const rest = new Rest(rp.defaults({
   auth: {
     bearer: process.env.TOKEN
   },
   baseUrl: `https://${process.env.DOMAIN}/api/v1/`,
-  timeout: Number.parseInt(process.env.TIMEOUT, undefined),
+  timeout: toInt(process.env.TIMEOUT),
   json: true
-}))
+}), new Limiter(toInt(process.env.LIMIT_COUNT), toInt(process.env.LIMIT_TIME)))
 export const rootStatus = process.env.ROOT_STATUS

src/tools/data/push.ts

@@ -9,13 +9,16 @@ async function run() {
     return
   }
 
-  const post = async (in_reply_to_id: string, status: string) => rest.postStatus({
+  const post = async (inReplyToId: string, status: string) => rest.postStatus({
-    in_reply_to_id, status,
+    in_reply_to_id: inReplyToId,
+    status,
     visibility: visibility as any || 'direct'
   }).then(s => s.id)
 
+  // TODO: push database.json
   const data = JSON.parse(fs.readFileSync(file, 'utf8'))
   const folder = await post(rootStatus, title || file)
+  Logger.info('Content length', data.length)
   for (const row of data) {
     if (typeof row === 'string') {
       await post(folder, row)
@@ -29,6 +32,6 @@ async function run() {
       throw new Error('bad type ' + typeof row)
     }
   }
-  Logger.info('data added', folder)
+  Logger.info('Data added', folder)
 }
 run()

src/tools/token/get.ts

@@ -0,0 +1,38 @@
+import querystring from 'querystring'
+import readline from 'readline'
+import rp from 'request-promise-native'
+import { rest } from '../../prepare'
+import Logger from '../../utils/Logger'
+
+async function run() {
+  try {
+    Logger.info('Checking current token')
+    const { acct: me } = await rest.getMe()
+    Logger.info(`Already logged as ${me}`)
+  } catch (error) {
+    Logger.info('Trying to get token')
+    const redirectUri = 'urn:ietf:wg:oauth:2.0:oob'
+    const scopes = 'read:accounts read:statuses write:statuses write:media'
+
+    const { client_id, client_secret } = await rest.postApp(redirectUri, scopes)
+
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: process.stdout
+    })
+    Logger.info(`Please visit: https://${process.env.DOMAIN}/oauth/authorize?scope=${querystring.escape(scopes)}&redirect_uri=${redirectUri}&response_type=code&client_id=${client_id} to validate your token`)
+    rl.question('And then paste token here: ', async code => {
+      const { access_token } = await rp.post(`https://${process.env.DOMAIN}/oauth/token`, {
+        json: true,
+        body: {
+          client_id, client_secret,
+          grant_type: 'authorization_code',
+          code, redirect_uri: redirectUri
+        }
+      })
+      rl.write(`Put TOKEN=${access_token} in .env`)
+      rl.close()
+    })
+  }
+}
+run()

src/types/config.ts

@@ -5,6 +5,7 @@ export interface RootConfig {
     async: boolean
     deep: boolean
     shared: boolean
+    file: boolean
 }
 export interface Action {
     id: string,
@@ -27,6 +28,7 @@ export interface ActionConfig {
 }
 
 export interface ActionConfigData {
+    file: boolean
     from: string[]
     deep: boolean
     shared: boolean

src/utils/Limiter.ts

@@ -0,0 +1,52 @@
+import Logger from "./Logger";
+
+export default class Limiter {
+
+  private past: number[] = []
+  private queue: any[] = []
+  private timer: NodeJS.Timeout | NodeJS.Immediate = null
+
+  constructor(private calls: number, private time: number, private sync: boolean = false) { }
+
+  dequeue() {
+    const now = Date.now()
+
+    this.past = this.past.filter(t => now - t < this.time)
+
+    while (this.past.length < this.calls && this.queue.length > 0) {
+      this.past.push(now)
+      const action = this.queue.shift()
+      action()
+
+      if (this.sync) {
+        break
+      }
+    }
+
+    if (this.queue.length <= 0) {
+      this.timer = null
+    } else {
+      const delay = this.sync ? this.time / this.calls : this.time - now + this.past[0]
+      Logger.warn(`Limiter: waiting ${delay}ms...`)
+      this.timer = setTimeout(this.dequeue.bind(this), delay)
+    }
+  }
+
+  limit(callback: CallableFunction) {
+    this.queue.push(callback)
+    if (this.timer === null) {
+      this.timer = setImmediate(this.dequeue.bind(this))
+    }
+  }
+
+  promise<T>(promise: Promise<T>) {
+    let res: (value?: T) => void = null
+    let rej: (reason?: any) => void = null
+    const wrapper = new Promise<T>((resolve, reject) => {
+      res = resolve
+      rej = reject
+    })
+    this.limit(() => promise.then(res).catch(rej))
+    return wrapper
+  }
+}

tsconfig.json

@@ -5,6 +5,7 @@
         "target": "es6",
         "noImplicitAny": true,
         "moduleResolution": "node",
+        "resolveJsonModule": true,
         "sourceMap": false,
         "outDir": "dist",
         "baseUrl": ".",