diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d37bab4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +name: CI + +on: [push, pull_request] + +env: + BINARY_PREFIX: "qinglong-go_" + BINARY_SUFFIX: "" + COMMIT_ID: "${{ github.sha }}" + PR_PROMPT: "::warning:: Build artifact will not be uploaded due to the workflow is trigged by pull request." + LD_FLAGS: "-w -s" + +jobs: + build: + name: Build binary CI + runs-on: ubuntu-latest + strategy: + matrix: + # build and publish in parallel: linux/386, linux/amd64, windows/386, windows/amd64, darwin/amd64, darwin/arm64 + goos: [linux, windows, darwin] + goarch: ["386", amd64, arm, arm64] + exclude: + - goos: darwin + goarch: arm + - goos: darwin + goarch: "386" + - goos: windows + goarch: arm64 + - goos: windows + goarch: arm + fail-fast: true + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Fetch all tags + run: git fetch --force --tags + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.18 + + - name: Build binary file + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + IS_PR: ${{ !!github.head_ref }} + CGO_ENABLED: 0 + run: | + if [ $GOOS = "windows" ]; then export BINARY_SUFFIX="$BINARY_SUFFIX.exe"; fi + if $IS_PR ; then echo $PR_PROMPT; fi + export BINARY_NAME="$BINARY_PREFIX$GOOS_$GOARCH$BINARY_SUFFIX" + export LD_FLAGS="-w -s -X github.com/huoxue1/qinglong-go/api/system.VERSION=$COMMIT_ID" + go mod tidy + go build -o "output/$BINARY_NAME" -trimpath -ldflags "$LD_FLAGS" ./ + - name: Upload artifact + uses: actions/upload-artifact@v2 + if: ${{ !github.head_ref }} + with: + name: ${{ matrix.goos }}_${{ matrix.goarch }} + path: output/ + + diff --git a/.github/workflows/push-to-docker.yml b/.github/workflows/push-to-docker.yml new file mode 100644 index 0000000..d22e2e3 --- /dev/null +++ b/.github/workflows/push-to-docker.yml @@ -0,0 +1,90 @@ +name: docker build + +on: + push: + branches: + - 'main' + tags: + - '*' + pull_request: + branches: + - 'main' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - + name: Fetch all tags + run: git fetch --force --tags + - + name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.18 + - + name: Cache Go modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - + name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v2 + - + name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + - + name: Check snapshot + if: "!startsWith(github.ref, 'refs/tags/')" + id: snapshot + run: echo '::set-output name=ARG::--snapshot' + - + name: Run GoReleaser + uses: goreleaser/goreleaser-action@v3 + with: + distribution: goreleaser + version: latest + args: build --rm-dist --id docker ${{ steps.snapshot.outputs.ARG }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - + name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: huoxue1/qinglong-go + tags: | + type=raw,value=latest + type=ref,event=tag + - + name: Docker Login + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v3 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + platforms: linux/amd64,linux/arm64,linux/386,linux/arm + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + shm-size: 2g + ulimit: core=0:0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7e6da7c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: release + +on: + push: + tags: + - 'v*' + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: '1.18' + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.TOKEN }} + diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..16f2593 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,94 @@ +linters-settings: + errcheck: + ignore: fmt:.*,io/ioutil:^Read.* + ignoretests: true + + goimports: + local-prefixes: github.com/huoxue1/qinglong-go + + gocritic: + disabled-checks: + - exitAfterDefer + + forbidigo: + # Forbid the following identifiers + forbid: + - ^fmt\.Errorf$ # consider errors.Errorf in github.com/pkg/errors + +linters: + # please, do not use `enable-all`: it's deprecated and will be removed soon. + # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint + fast: true + enable: + - bodyclose + - deadcode + - depguard + - dogsled + - dupl + - errcheck + - exportloopref + - exhaustive + #- funlen + #- goconst + - gocritic + #- gocyclo + - gofmt + - goimports + - goprintffuncname + #- gosec + - gosimple + - govet + - ineffassign + - misspell + - nolintlint + - rowserrcheck + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace + - prealloc + - predeclared + - asciicheck + - forbidigo + - makezero + - revive + #- interfacer + + # don't enable: + # - scopelint + # - gochecknoglobals + # - gocognit + # - godot + # - godox + # - goerr113 + # - interfacer + # - maligned + # - nestif + # - testpackage + # - wsl + +run: + # default concurrency is a available CPU number. + # concurrency: 4 # explicitly omit this value to fully utilize available resources. + deadline: 5m + issues-exit-code: 1 + tests: false + +# output configuration options +output: + format: "colored-line-number" + print-issued-lines: true + print-linter-name: true + uniq-by-line: true + +issues: + # Fix found issues (if it's supported by the linter) + fix: true + exclude-use-default: false + exclude: + - "Error return value of .((os.)?std(out|err)..*|.*Close|.*Flush|os.Remove(All)?|.*print(f|ln)?|os.(Un)?Setenv). is not check" diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..890a47f --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,108 @@ +env: + - GO111MODULE=on +before: + hooks: + - go mod tidy +builds: + + - id: nowin + env: + - CGO_ENABLED=0 + - GO111MODULE=on + goos: + - linux + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + goarm: + - 7 + ignore: + - goos: darwin + goarch: arm + - goos: darwin + goarch: 386 + - goos: windows + goarch: arm + mod_timestamp: "{{ .CommitTimestamp }}" + flags: + - -trimpath + ldflags: + - -s -w -X main.VERSION=v{{.Version}} + - id: win + env: + - CGO_ENABLED=0 + - GO111MODULE=on + goos: + - windows + goarch: + - 386 + - amd64 + goarm: + - 7 + mod_timestamp: "{{ .CommitTimestamp }}" + flags: + - -trimpath + ldflags: + - -s -w -X main.VERSION=v{{.Version}} + - id: docker + env: + - CGO_ENABLED=0 + - GO111MODULE=on + goos: + - linux + goarch: + - amd64 + - arm64 + - 386 + - arm + mod_timestamp: "{{ .CommitTimestamp }}" + flags: + - -trimpath + ldflags: + - -s -w -X main.VERSION=v{{.Version}} + +checksum: + name_template: "{{ .ProjectName }}_checksums.txt" +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - fix typo + - Merge pull request + - Merge branch + - Merge remote-tracking + - go mod tidy + +archives: + - id: binary + builds: + - win + name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + format_overrides: + - goos: windows + format: binary + - id: nowin + builds: + - nowin + - win + name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" + format_overrides: + - goos: windows + format: zip + +nfpms: + - license: AGPL 3.0 + homepage: https://github.com/johlanse/study_xxqg + file_name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + formats: + - deb + - rpm + maintainer: johlanse + builds: + - nowin + - win \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..26a4e5d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ + +FROM python:alpine + +LABEL maintainer="${QL_MAINTAINER}" + +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/share/pnpm/global/5/node_modules \ + LANG=zh_CN.UTF-8 \ + SHELL=/bin/bash \ + PS1="\u@\h:\w \$ " \ + QL_DIR=/ql \ + QL_BRANCH=${QL_BRANCH} + +WORKDIR ${QL_DIR} + +RUN set -x \ + && sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories \ + && apk update -f \ + && apk upgrade \ + && apk --no-cache add -f bash \ + coreutils \ + moreutils \ + git \ + curl \ + wget \ + tzdata \ + perl \ + openssl \ + nodejs \ + jq \ + openssh \ + npm \ + && rm -rf /var/cache/apk/* \ + && apk update \ + && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \ + && echo "Asia/Shanghai" > /etc/timezone \ + && git config --global user.email "qinglong@@users.noreply.github.com" \ + && git config --global user.name "qinglong" \ + && git config --global http.postBuffer 524288000 \ + && npm install -g yarn \ + && rm -rf /root/.cache \ + && rm -rf /root/.npm \ + +COPY ./dist/docker_linux_$TARGETARCH*/qinglong-go ${QL_DIR}/qinglong-go + +RUN chmod -R 777 ${QL_DIR}/qinglong-go + +EXPOSE 8080 + +VOLUME ${QL_DIR}/data + +CMD cd ${QL_DIR} && ./qinglong-go \ No newline at end of file diff --git a/api/system/system.go b/api/system/system.go index 557f20a..719a6ec 100644 --- a/api/system/system.go +++ b/api/system/system.go @@ -8,6 +8,10 @@ import ( "path" ) +var ( + VERSION = "UNKNOWN" +) + func Api(group *gin.RouterGroup) { group.GET("", get()) } @@ -18,7 +22,7 @@ func get() gin.HandlerFunc { exist := os.IsNotExist(err) ctx.JSON(200, res.Ok(system.System{ IsInitialized: !exist, - Version: "2.0.14", + Version: VERSION, LastCommitTime: "", LastCommitId: "", Branch: "master", diff --git a/api/user/package_sample.json b/api/user/package_sample.json new file mode 100644 index 0000000..e69de29 diff --git a/api/user/user.go b/api/user/user.go index 97597fc..ea9db28 100644 --- a/api/user/user.go +++ b/api/user/user.go @@ -16,6 +16,9 @@ import ( //go:embed config_sample.sh var sample []byte +//go:embed package_sample.json +var pack []byte + func Api(group *gin.RouterGroup) { group.GET("/", get()) group.PUT("/init", appInit()) @@ -45,8 +48,11 @@ func appInit() gin.HandlerFunc { _ = os.MkdirAll(path.Join("data", "log"), 0666) _ = os.MkdirAll(path.Join("data", "repo"), 0666) _ = os.MkdirAll(path.Join("data", "scripts"), 0666) + _ = os.MkdirAll(path.Join("data", "deps"), 0666) + _ = os.MkdirAll(path.Join("data", "raw"), 0666) _ = os.WriteFile(path.Join("data", "config", "config.sh"), sample, 0666) - _ = os.WriteFile(path.Join("data", "config", "config_sample.sh"), sample, 0666) + _ = os.WriteFile(path.Join("data", "scripts", "package.json"), pack, 0666) + _ = os.WriteFile(path.Join("data", "config", "config.sample.sh"), sample, 0666) type Req struct { UserName string `json:"username"` Password string `json:"password"` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..3156442 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.5" +services: + xuexi-auto: + image: huoxue1/qinglong-go:latest + # 容器名 + container_name: qinglong-go + environment: + # 时区 + - TZ=Asia/Shanghai + # 配置文件路径 + volumes: + - ./data:/ql/data + # 映射端口 + ports: + - 8080:8080 + restart: unless-stopped \ No newline at end of file diff --git a/main.go b/main.go index 3be5008..9a92fdb 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( nested "github.com/Lyrics-you/sail-logrus-formatter/sailor" "github.com/huoxue1/qinglong-go/controller" + "github.com/huoxue1/qinglong-go/service" rotates "github.com/lestrrat-go/file-rotatelogs" log "github.com/sirupsen/logrus" "io" @@ -12,7 +13,7 @@ import ( ) func init() { - w, err := rotates.New(path.Join("data", "logs", "%Y-%m-%d.log"), rotates.WithRotationTime(time.Hour*24)) + w, err := rotates.New(path.Join("data", "log", "qinglong-go", "%Y-%m-%d.log"), rotates.WithRotationTime(time.Hour*24)) if err != nil { log.Errorf("rotates init err: %v", err) panic(err) @@ -36,6 +37,7 @@ func init() { } func main() { + service.AppInit() engine := controller.Router() _ = engine.Run(":8080") } diff --git a/models/Crontabs.go b/models/Crontabs.go index f0cb90e..b13f73f 100644 --- a/models/Crontabs.go +++ b/models/Crontabs.go @@ -48,6 +48,13 @@ func QueryRunningCron() ([]*Crontabs, error) { return crontabs, err } +func QueryCronByDir(dir string) ([]*Crontabs, error) { + crontabs := make([]*Crontabs, 0) + session := engine.Table(new(Crontabs)).Where(builder.Like{"command", "task " + dir + "%"}) + err := session.Find(&crontabs) + return crontabs, err +} + func FindAllEnableCron() []*Crontabs { crontabs := make([]*Crontabs, 0) err := engine.Table(new(Crontabs)).Where("isdisabled=?", 0).Find(&crontabs) diff --git a/service/app.go b/service/app.go new file mode 100644 index 0000000..f4a643e --- /dev/null +++ b/service/app.go @@ -0,0 +1,29 @@ +package service + +import ( + "context" + "github.com/huoxue1/qinglong-go/utils" + log "github.com/sirupsen/logrus" + "os" + "path" +) + +func AppInit() { + go runYarn() +} + +func runYarn() { + defer func() { + recover() + }() + _, err := os.Stat(path.Join("data", "scripts", "package.json")) + if os.IsNotExist(err) { + return + } + ch := make(chan int, 1) + utils.RunTask(context.WithValue(context.Background(), "cancel", ch), "yarn install", map[string]string{}, func(ctx context.Context) { + log.Infoln("开始执行yarn初始化！") + }, func(ctx context.Context) { + log.Infoln("yarn初始化执行完成！") + }, os.Stdout) +} diff --git a/service/cron/manager.go b/service/cron/manager.go index 544f709..3697011 100644 --- a/service/cron/manager.go +++ b/service/cron/manager.go @@ -129,16 +129,28 @@ func runCron(crontabs *models.Crontabs) { } func AddTask(crontabs *models.Crontabs) { - c := cron.New() + crons := strings.Split(crontabs.Schedule, " ") + var c *cron.Cron + if len(crons) == 5 { + c = cron.New() + + } else if len(crons) == 6 { + c = cron.New(cron.WithParser( + cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))) + } else { + log.Errorf("the task %s cron %s is error", crontabs.Name, crontabs.Command) + return + } _, err := c.AddFunc(crontabs.Schedule, func() { runCron(crontabs) }) if err != nil { - log.Errorln("添加task错误" + err.Error()) + log.Errorln("添加task错误" + crontabs.Schedule + err.Error()) return } c.Start() manager.Store(crontabs.Id, c) + } func handCommand(command string) *task { @@ -189,32 +201,3 @@ func handCommand(command string) *task { } return ta } - -//type myWriter struct { -// fileName string -//} -// -//func (m *myWriter) Write(p []byte) (n int, err error) { -// file, _ := os.OpenFile(m.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) -// n, err = file.Write(p) -// file.Close() -// return n, err -//} -// -////通过管道同步获取日志的函数 -//func syncLog(reader io.ReadCloser, writer io.Writer) { -// buf := make([]byte, 1) -// for { -// strNum, err := reader.Read(buf) -// if strNum > 0 { -// outputByte := buf[:strNum] -// writer.Write(outputByte) -// } -// if err != nil { -// //读到结尾 -// if err == io.EOF || strings.Contains(err.Error(), "file already closed") { -// return -// } -// } -// } -//} diff --git a/service/subscription/manager.go b/service/subscription/manager.go index 4c9b041..d48a563 100644 --- a/service/subscription/manager.go +++ b/service/subscription/manager.go @@ -8,6 +8,7 @@ import ( "github.com/huoxue1/qinglong-go/service/config" "github.com/huoxue1/qinglong-go/service/cron" "github.com/huoxue1/qinglong-go/utils" + log "github.com/sirupsen/logrus" "io" "os" "os/exec" @@ -45,12 +46,60 @@ func downloadFiles(subscriptions *models.Subscriptions) { if err != nil { return } - addScripts(subscriptions) + if config.GetKey("AutoAddCron") == "true" { + addScripts(subscriptions) + } else { + log.Infoln("未配置自动添加定时任务，不添加任务！") + } + file, _ := os.OpenFile(subscriptions.LogPath, os.O_APPEND|os.O_RDWR, 0666) file.WriteString(fmt.Sprintf("\n##执行结束.. %s，耗时0秒\n\n", time.Now().Format("2006-01-02 15:04:05"))) _ = file.Close() subscriptions.Status = 1 models.UpdateSubscription(subscriptions) + } else if subscriptions.Type == "file" { + addRawFiles(subscriptions) + } +} + +func addRawFiles(subscriptions *models.Subscriptions) { + subscriptions.LogPath = "data/log/" + time.Now().Format("2006-01-02") + "/" + subscriptions.Alias + "_" + uuid.New().String() + ".log" + subscriptions.Status = 0 + file, _ := os.OpenFile(subscriptions.LogPath, os.O_CREATE|os.O_RDWR, 0666) + defer file.Close() + _ = models.UpdateSubscription(subscriptions) + defer func() { + subscriptions.Status = 1 + _ = models.UpdateSubscription(subscriptions) + }() + err := utils.DownloadFile(subscriptions.Url, path.Join("data", "raw", subscriptions.Alias)) + if err != nil { + _, _ = file.WriteString(err.Error() + "\n") + return + } + name, c, err := getSubCron(path.Join("data", "raw", subscriptions.Alias)) + if err != nil { + _, _ = file.WriteString(err.Error() + "\n") + return + } + utils.Copy(path.Join("data", "raw", subscriptions.Alias), path.Join("data", "scripts", subscriptions.Alias)) + if c != "" { + command, err := models.GetCronByCommand(fmt.Sprintf("task %s", subscriptions.Alias)) + if err != nil { + file.WriteString("已添加新的定时任务 " + name + "\n") + _, _ = cron.AddCron(&models.Crontabs{ + Name: name, + Command: fmt.Sprintf("task %s", subscriptions.Alias), + Schedule: c, + Timestamp: time.Now().Format("Mon Jan 02 2006 15:04:05 MST"), + Status: 1, + Labels: []string{}, + }) + } else { + command.Name = name + command.Schedule = c + _ = cron.UpdateCron(command) + } } } @@ -98,6 +147,11 @@ func addScripts(subscriptions *models.Subscriptions) { if err != nil { return } + crontabs, _ := models.QueryCronByDir(subscriptions.Alias) + cronMap := make(map[string]*models.Crontabs, len(crontabs)) + for _, crontab := range crontabs { + cronMap[crontab.Command] = crontab + } for _, entry := range dir { // 判断文件后缀 if !utils.In(strings.TrimPrefix(filepath.Ext(entry.Name()), "."), extensions) { @@ -128,6 +182,7 @@ func addScripts(subscriptions *models.Subscriptions) { command.Name = name command.Schedule = c _ = cron.UpdateCron(command) + delete(cronMap, command.Command) } } @@ -139,6 +194,12 @@ func addScripts(subscriptions *models.Subscriptions) { } } } + if config.GetKey("AutoDelCron") == "true" { + for _, m := range cronMap { + file.WriteString("已删除失效的任务 " + m.Name + "\n") + models.DeleteCron(m.Id) + } + } } func getSubCron(filePath string) (name string, cron string, err error) { @@ -159,5 +220,6 @@ func getSubCron(filePath string) (name string, cron string, err error) { } else { return "", "", errors.New("not found cron") } + cron = strings.TrimPrefix(cron, " ") return } diff --git a/utils/file.go b/utils/file.go index f1449fb..97348d1 100644 --- a/utils/file.go +++ b/utils/file.go @@ -3,6 +3,7 @@ package utils import ( log "github.com/sirupsen/logrus" "io" + "net/http" "os" "path" "path/filepath" @@ -48,3 +49,31 @@ func Copy(src, dest string) { } } } + +func DownloadFile(url, filePath string) error { + response, err := http.Get(url) + if err != nil { + return err + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + + } + }(response.Body) + file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return err + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + + } + }(file) + _, err = io.Copy(file, response.Body) + if err != nil { + return err + } + return nil +}