first commit

.gitignore

@@ -0,0 +1,3 @@
+target
+node_modules
+.wrangler

Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "miniflux-ai"
+version = "0.1.0"
+edition = "2021"
+authors = [ "zhu327" ]
+
+[package.metadata.release]
+release = false
+
+# https://github.com/rustwasm/wasm-pack/issues/1247
+[package.metadata.wasm-pack.profile.release]
+wasm-opt = false
+
+[profile.release]
+lto = true
+strip = true
+codegen-units = 1
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+base64 = "0.21"
+worker = { version="0.3.4" }
+worker-macros = { version="0.3.4" }
+console_error_panic_hook = { version = "0.1.7" }
+reqwest = { version = "0.11", features = ["json"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+futures = "0.3"

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Timmy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,57 @@
+# Miniflux AI Summarizer
+
+This Cloudflare Workers tool automatically adds AI-generated summaries to articles in your Miniflux RSS reader. The summaries are generated using the OpenAI API and appended to articles in a user-friendly format.
+
+## Features
+
+- **Automated Summarization**: Fetches unread articles from Miniflux, generates concise summaries using AI, and updates the articles with the summaries.
+- **Customizable**: Configure the list of whitelisted websites, API endpoints, and AI model parameters through environment variables.
+- **Concurrency**: Uses asynchronous Rust features to handle multiple articles concurrently, ensuring quick processing.
+- **Cloudflare Integration**: Deployed as a serverless function on Cloudflare Workers, leveraging the scalability and performance of Cloudflare's global network.
+- **Recommended Model**: Uses the Cloudflare Workers AI model `@cf/qwen/qwen1.5-14b-chat-awq` for generating high-quality, concise summaries.
+
+## Getting Started
+
+### Prerequisites
+
+- [Rust](https://www.rust-lang.org/tools/install) installed
+- A Miniflux instance with API access
+- An OpenAI account with access to the model endpoint
+- A Cloudflare account
+
+### Installation
+
+1. Clone the repository:
+   ```bash
+   git clone https://github.com/yourusername/miniflux-ai.git
+   cd miniflux-ai
+   ```
+
+2. Deploy to Cloudflare Workers:
+   ```bash
+   npx wrangler deploy
+   ```
+
+### Configuration
+
+The tool is configured using environment variables, which are set in the `wrangler.toml` file:
+
+- `MINIFLUX_URL`: Your Miniflux instance URL.
+- `MINIFLUX_USERNAME`: Your Miniflux username.
+- `MINIFLUX_PASSWORD`: Your Miniflux password.
+- `OPENAI_URL`: The endpoint for the OpenAI API.
+- `OPENAI_TOKEN`: Your OpenAI API token.
+- `OPENAI_MODEL`: The model ID to use for generating summaries. We recommend using the `@cf/qwen/qwen1.5-14b-chat-awq` model for best results.
+- `WHITELIST_URL`: A comma-separated list of website URLs that should be summarized.
+
+### Usage
+
+The tool runs as a scheduled Cloudflare Worker, querying for unread articles every 5 minutes. If an article is from a whitelisted site and does not contain code blocks, it generates a summary and updates the article.
+
+### Contributing
+
+Contributions are welcome! Please feel free to submit issues, feature requests, or pull requests.
+
+### License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

src/lib.rs

@@ -0,0 +1,248 @@
+use base64;
+use futures::future::join_all;
+use reqwest::header::{AUTHORIZATION, CONTENT_TYPE};
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use worker::{event, Env, ScheduleContext, ScheduledEvent};
+
+#[derive(Debug, Deserialize)]
+struct Feed {
+    site_url: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct Entry {
+    id: u64,
+    content: String,
+    feed: Feed,
+}
+
+#[derive(Debug, Deserialize)]
+struct ApiResponse {
+    entries: Vec<Entry>,
+}
+
+#[derive(Serialize)]
+struct UpdateRequest {
+    content: String,
+}
+
+async fn get_entries(
+    base_url: &str,
+    username: &str,
+    password: &str,
+) -> Result<ApiResponse, Box<dyn std::error::Error>> {
+    // 创建 HTTP 客户端
+    let client = reqwest::Client::new();
+
+    // 使用 Basic Auth 进行身份验证
+    let auth = format!(
+        "Basic {}",
+        base64::encode(format!("{}:{}", username, password))
+    );
+
+    // 发送 GET 请求
+    let response = client
+        .get(&format!("{}/v1/entries?status=unread&limit=100", base_url))
+        .header(AUTHORIZATION, auth)
+        .header(CONTENT_TYPE, "application/json")
+        .send()
+        .await?
+        .json::<ApiResponse>()
+        .await?;
+
+    Ok(response)
+}
+
+async fn update_entry(
+    base_url: &str,
+    username: &str,
+    password: &str,
+    id: u64,
+    content: &str,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let client = reqwest::Client::new();
+    let auth = format!(
+        "Basic {}",
+        base64::encode(format!("{}:{}", username, password))
+    );
+
+    let url = format!("{}/v1/entries/{}", base_url, id);
+    let update_request = UpdateRequest {
+        content: content.to_string(),
+    };
+
+    client
+        .put(&url)
+        .header(AUTHORIZATION, auth)
+        .header(CONTENT_TYPE, "application/json")
+        .json(&update_request) // 将请求体序列化为 JSON
+        .send()
+        .await?
+        .error_for_status()?;
+
+    Ok(())
+}
+
+#[derive(Serialize)]
+struct ChatCompletionRequest {
+    model: String,
+    messages: Vec<Message>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Message {
+    role: String,
+    content: String,
+}
+
+#[derive(Deserialize)]
+struct ChatCompletionChoice {
+    message: Message,
+}
+
+#[derive(Deserialize)]
+struct ChatCompletionResponse {
+    choices: Vec<ChatCompletionChoice>,
+}
+
+async fn request_openai_chat_completion(
+    base_url: &str,
+    api_key: &str,
+    model: &str,
+    messages: Vec<Message>,
+) -> Result<String, Box<dyn std::error::Error>> {
+    let client = reqwest::Client::new();
+    let request_body = ChatCompletionRequest {
+        model: model.to_string(),
+        messages,
+    };
+
+    let response = client
+        .post(&format!("{}/v1/chat/completions", base_url))
+        .header(AUTHORIZATION, format!("Bearer {}", api_key))
+        .header(CONTENT_TYPE, "application/json")
+        .json(&request_body)
+        .send()
+        .await?;
+
+    if response.status().is_success() {
+        let completion_response: ChatCompletionResponse = response.json().await?;
+        Ok(completion_response.choices[0].message.content.clone())
+    } else {
+        let error_message = response.text().await?;
+        Err(format!("Error: {:?}", error_message).into())
+    }
+}
+
+struct Miniflux {
+    url: String,
+    username: String,
+    password: String,
+}
+
+struct OpenAi {
+    url: String,
+    token: String,
+    model: String,
+}
+
+struct Config {
+    miniflux: Miniflux,
+    openai: OpenAi,
+    whitelist: HashSet<String>,
+}
+
+async fn generate_and_update_entry(
+    config: &Config,
+    entry: Entry,
+) -> Result<(), Box<dyn std::error::Error>> {
+    let content: &str = &entry.content;
+    // Check if the content should be summarized and if the site is whitelisted
+    if content.starts_with("<pre") || !config.whitelist.contains(&entry.feed.site_url) {
+        return Ok(());
+    }
+
+    let messages = vec![
+        Message {
+            role: "system".to_string(),
+            content: "Please summarize the content of the article under 50 words in Chinese. Do not add any additional Character、markdown language to the result text. 请用不超过50个汉字概括文章内容。结果文本中不要添加任何额外的字符、Markdown语言。".to_string(),
+        },
+        Message {
+            role: "user".to_string(),
+            content: format!(
+                "The following is the input content:\n---\n {}",
+                content,
+            ),
+        },
+    ];
+
+    // Generate summary
+    if let Ok(summary) = request_openai_chat_completion(
+        &config.openai.url,
+        &config.openai.token,
+        &config.openai.model,
+        messages,
+    )
+    .await
+    {
+        let updated_content = format!(
+            "<pre style=\"white-space: pre-wrap;\"><code>\n💡AI 摘要：\n{}</code></pre><hr><br />{}",
+            summary, content
+        );
+
+        // Update the entry
+        update_entry(
+            &config.miniflux.url,
+            &config.miniflux.username,
+            &config.miniflux.password,
+            entry.id,
+            &updated_content,
+        )
+        .await?;
+    }
+
+    Ok(())
+}
+
+#[event(scheduled)]
+async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) {
+    let config = &Config {
+        whitelist: env
+            .var("WHITELIST_URL")
+            .unwrap()
+            .to_string()
+            .split(",")
+            .map(|s| s.to_string())
+            .collect(),
+        openai: OpenAi {
+            url: env.var("OPENAI_URL").unwrap().to_string(),
+            token: env.var("OPENAI_TOKEN").unwrap().to_string(),
+            model: env.var("OPENAI_MODEL").unwrap().to_string(),
+        },
+        miniflux: Miniflux {
+            url: env.var("MINIFLUX_URL").unwrap().to_string(),
+            username: env.var("MINIFLUX_USERNAME").unwrap().to_string(),
+            password: env.var("MINIFLUX_PASSWORD").unwrap().to_string(),
+        },
+    };
+
+    // 查询未读文章
+    let entries = get_entries(
+        &config.miniflux.url,
+        &config.miniflux.username,
+        &config.miniflux.password,
+    )
+    .await
+    .unwrap();
+
+    // 生成摘要并更新的并发任务
+    let tasks: Vec<_> = entries
+        .entries
+        .into_iter()
+        .map(|entry| generate_and_update_entry(&config, entry))
+        .collect();
+
+    // 执行所有任务并等待结果
+    join_all(tasks).await;
+}

wrangler.toml

@@ -0,0 +1,18 @@
+name = "miniflux-ai"
+main = "build/worker/shim.mjs"
+compatibility_date = "2024-08-06"
+
+[build]
+command = "cargo install -q worker-build && worker-build --release"
+
+[triggers]
+crons = ["*/5 * * * *"]
+
+[vars]
+MINIFLUX_URL = "your miniflux url"
+MINIFLUX_USERNAME = "your miniflux username"
+MINIFLUX_PASSWORD = "your miniflux password"
+OPENAI_URL = "https://api.cloudflare.com/client/v4/accounts/{your cloudflare account}/ai"
+OPENAI_TOKEN = "your cloudflare workers AI token"
+OPENAI_MODEL = "@cf/qwen/qwen1.5-14b-chat-awq"
+WHITELIST_URL = "https://www.zaobao.com/news/china,https://t.me/s/theinitium_rss,https://cn.nytimes.com,https://www.latepost.com/news/get-news-data,https://t.me/s/wsj_rss,http://www.zhihu.com,https://new.qq.com/omn/author/5157372,https://www.huxiu.com"