Initial commit

.github/workflows/build-api-image.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SHA=$(git rev-parse HEAD)
+REPO_NAME=langgenius/dify
+API_REPO_NAME="${REPO_NAME}-api"
+
+if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
+  LATEST_TAG="pr-${PR_NUM}"
+  CACHE_FROM_TAG="latest"
+elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="latest"
+else
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="${REFSPEC}"
+fi
+
+if [[ "${REFSPEC}" == "main" ]]; then
+  LATEST_TAG="latest"
+  CACHE_FROM_TAG="latest"
+fi
+
+echo "Pulling cache image ${API_REPO_NAME}:${CACHE_FROM_TAG}"
+if docker pull "${API_REPO_NAME}:${CACHE_FROM_TAG}"; then
+  API_CACHE_FROM_SCRIPT="--cache-from ${API_REPO_NAME}:${CACHE_FROM_TAG}"
+else
+  echo "WARNING: Failed to pull ${API_REPO_NAME}:${CACHE_FROM_TAG}, disable build image cache."
+  API_CACHE_FROM_SCRIPT=""
+fi
+
+
+cat<<EOF
+  Rolling with tags:
+  - ${API_REPO_NAME}:${SHA}
+  - ${API_REPO_NAME}:${REFSPEC}
+  - ${API_REPO_NAME}:${LATEST_TAG}
+EOF
+
+#
+# Build image
+#
+cd api
+docker build \
+  ${API_CACHE_FROM_SCRIPT} \
+  --build-arg COMMIT_SHA=${SHA} \
+  -t "${API_REPO_NAME}:${SHA}" \
+  -t "${API_REPO_NAME}:${REFSPEC}" \
+  -t "${API_REPO_NAME}:${LATEST_TAG}" \
+  --label "sha=${SHA}" \
+  --label "built_at=$(date)" \
+  --label "build_actor=${GITHUB_ACTOR}" \
+  .
+
+# push
+docker push --all-tags "${API_REPO_NAME}"

.github/workflows/build-api-image.yml

@@ -0,0 +1,43 @@
+name: Build and Push API Image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'deploy/dev'
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    steps:
+    - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USER }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push Docker image
+      shell: bash
+      env:
+        DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+        DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        /bin/bash .github/workflows/build-api-image.sh
+
+    - name: Deploy to server
+      if: github.ref == 'refs/heads/deploy/dev'
+      uses: appleboy/ssh-action@v0.1.8
+      with:
+        host: ${{ secrets.SSH_HOST }}
+        username: ${{ secrets.SSH_USER }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script: |
+          ${{ secrets.SSH_SCRIPT }}

.github/workflows/build-web-image.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SHA=$(git rev-parse HEAD)
+REPO_NAME=langgenius/dify
+WEB_REPO_NAME="${REPO_NAME}-web"
+
+if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
+  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  PR_NUM=$(echo "${GITHUB_REF}" | sed 's:refs/pull/::' | sed 's:/merge::')
+  LATEST_TAG="pr-${PR_NUM}"
+  CACHE_FROM_TAG="latest"
+elif [[ "${GITHUB_EVENT_NAME}" == "release" ]]; then
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/tags/::' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="latest"
+else
+  REFSPEC=$(echo "${GITHUB_REF}" | sed 's:refs/heads/::' | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
+  LATEST_TAG="${REFSPEC}"
+  CACHE_FROM_TAG="${REFSPEC}"
+fi
+
+if [[ "${REFSPEC}" == "main" ]]; then
+  LATEST_TAG="latest"
+  CACHE_FROM_TAG="latest"
+fi
+
+echo "Pulling cache image ${WEB_REPO_NAME}:${CACHE_FROM_TAG}"
+if docker pull "${WEB_REPO_NAME}:${CACHE_FROM_TAG}"; then
+  WEB_CACHE_FROM_SCRIPT="--cache-from ${WEB_REPO_NAME}:${CACHE_FROM_TAG}"
+else
+  echo "WARNING: Failed to pull ${WEB_REPO_NAME}:${CACHE_FROM_TAG}, disable build image cache."
+  WEB_CACHE_FROM_SCRIPT=""
+fi
+
+
+cat<<EOF
+  Rolling with tags:
+  - ${WEB_REPO_NAME}:${SHA}
+  - ${WEB_REPO_NAME}:${REFSPEC}
+  - ${WEB_REPO_NAME}:${LATEST_TAG}
+EOF
+
+#
+# Build image
+#
+cd web
+docker build \
+  ${WEB_CACHE_FROM_SCRIPT} \
+  --build-arg COMMIT_SHA=${SHA} \
+  -t "${WEB_REPO_NAME}:${SHA}" \
+  -t "${WEB_REPO_NAME}:${REFSPEC}" \
+  -t "${WEB_REPO_NAME}:${LATEST_TAG}" \
+  --label "sha=${SHA}" \
+  --label "built_at=$(date)" \
+  --label "build_actor=${GITHUB_ACTOR}" \
+  .
+
+docker push --all-tags "${WEB_REPO_NAME}"

.github/workflows/build-web-image.yml

@@ -0,0 +1,43 @@
+name: Build and Push WEB Image
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'deploy/dev'
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+    steps:
+    - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USER }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push Docker image
+      shell: bash
+      env:
+        DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+        DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+      run: |
+        /bin/bash .github/workflows/build-web-image.sh
+
+    - name: Deploy to server
+      if: github.ref == 'refs/heads/deploy/dev'
+      uses: appleboy/ssh-action@v0.1.8
+      with:
+        host: ${{ secrets.SSH_HOST }}
+        username: ${{ secrets.SSH_USER }}
+        key: ${{ secrets.SSH_PRIVATE_KEY }}
+        script: |
+          ${{ secrets.SSH_SCRIPT }}

.github/workflows/check_no_chinese_comments.py

@@ -0,0 +1,35 @@
+import os
+import re
+from zhon.hanzi import punctuation
+
+def has_chinese_characters(text):
+    for char in text:
+        if '\u4e00' <= char <= '\u9fff' or char in punctuation:
+            return True
+    return False
+
+def check_file_for_chinese_comments(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        for line_number, line in enumerate(file, start=1):
+            if has_chinese_characters(line):
+                print(f"Found Chinese characters in {file_path} on line {line_number}:")
+                print(line.strip())
+                return True
+    return False
+
+def main():
+    has_chinese = False
+    excluded_files = ["model_template.py", 'stopwords.py', 'commands.py', 'indexing_runner.py']
+
+    for root, _, files in os.walk("."):
+        for file in files:
+            if file.endswith(".py") and file not in excluded_files:
+                file_path = os.path.join(root, file)
+                if check_file_for_chinese_comments(file_path):
+                    has_chinese = True
+
+    if has_chinese:
+        raise Exception("Found Chinese characters in Python files. Please remove them.")
+
+if __name__ == "__main__":
+    main()

.github/workflows/check_no_chinese_comments.yml

@@ -0,0 +1,31 @@
+name: Check for Chinese comments
+
+on:
+  push:
+    branches:
+      - 'main'
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  check-chinese-comments:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install zhon
+
+    - name: Run script to check for Chinese comments
+      run: |
+        python .github/workflows/check_no_chinese_comments.py

.github/workflows/flake8.yml

@@ -0,0 +1,19 @@
+name: PEP8 Check
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+jobs:
+  pep8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Install flake8
+        run: pip install flake8
+      - name: Run flake8
+        run: flake8 --ignore=E501 .