添加功能

This commit is contained in:
3343780376 2022-11-26 10:31:26 +08:00
parent 44c7487099
commit e0717af09b
16 changed files with 608 additions and 35 deletions

63
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,63 @@
name: CI
on: [push, pull_request]
env:
BINARY_PREFIX: "qinglong-go_"
BINARY_SUFFIX: ""
COMMIT_ID: "${{ github.sha }}"
PR_PROMPT: "::warning:: Build artifact will not be uploaded due to the workflow is trigged by pull request."
LD_FLAGS: "-w -s"
jobs:
build:
name: Build binary CI
runs-on: ubuntu-latest
strategy:
matrix:
# build and publish in parallel: linux/386, linux/amd64, windows/386, windows/amd64, darwin/amd64, darwin/arm64
goos: [linux, windows, darwin]
goarch: ["386", amd64, arm, arm64]
exclude:
- goos: darwin
goarch: arm
- goos: darwin
goarch: "386"
- goos: windows
goarch: arm64
- goos: windows
goarch: arm
fail-fast: true
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Fetch all tags
run: git fetch --force --tags
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.18
- name: Build binary file
env:
GOOS: ${{ matrix.goos }}
GOARCH: ${{ matrix.goarch }}
IS_PR: ${{ !!github.head_ref }}
CGO_ENABLED: 0
run: |
if [ $GOOS = "windows" ]; then export BINARY_SUFFIX="$BINARY_SUFFIX.exe"; fi
if $IS_PR ; then echo $PR_PROMPT; fi
export BINARY_NAME="$BINARY_PREFIX$GOOS_$GOARCH$BINARY_SUFFIX"
export LD_FLAGS="-w -s -X github.com/huoxue1/qinglong-go/api/system.VERSION=$COMMIT_ID"
go mod tidy
go build -o "output/$BINARY_NAME" -trimpath -ldflags "$LD_FLAGS" ./
- name: Upload artifact
uses: actions/upload-artifact@v2
if: ${{ !github.head_ref }}
with:
name: ${{ matrix.goos }}_${{ matrix.goarch }}
path: output/

90
.github/workflows/push-to-docker.yml vendored Normal file
View File

@ -0,0 +1,90 @@
name: docker build
on:
push:
branches:
- 'main'
tags:
- '*'
pull_request:
branches:
- 'main'
jobs:
build:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
-
name: Fetch all tags
run: git fetch --force --tags
-
name: Set up Go
uses: actions/setup-go@v3
with:
go-version: 1.18
-
name: Cache Go modules
uses: actions/cache@v3
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
-
name: Set up QEMU
id: qemu
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v2
-
name: Check snapshot
if: "!startsWith(github.ref, 'refs/tags/')"
id: snapshot
run: echo '::set-output name=ARG::--snapshot'
-
name: Run GoReleaser
uses: goreleaser/goreleaser-action@v3
with:
distribution: goreleaser
version: latest
args: build --rm-dist --id docker ${{ steps.snapshot.outputs.ARG }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: huoxue1/qinglong-go
tags: |
type=raw,value=latest
type=ref,event=tag
-
name: Docker Login
if: github.event_name != 'pull_request'
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
name: Build and push
id: docker_build
uses: docker/build-push-action@v3
with:
context: .
push: ${{ github.event_name != 'pull_request' }}
platforms: linux/amd64,linux/arm64,linux/386,linux/arm
tags: ${{ steps.meta.outputs.tags }}
cache-from: type=gha
cache-to: type=gha,mode=max
shm-size: 2g
ulimit: core=0:0

29
.github/workflows/release.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: release
on:
push:
tags:
- 'v*'
jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2.3.4
with:
fetch-depth: 0
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: '1.18'
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v2
with:
version: latest
args: release --rm-dist
env:
GITHUB_TOKEN: ${{ secrets.TOKEN }}

94
.golangci.yml Normal file
View File

@ -0,0 +1,94 @@
linters-settings:
errcheck:
ignore: fmt:.*,io/ioutil:^Read.*
ignoretests: true
goimports:
local-prefixes: github.com/huoxue1/qinglong-go
gocritic:
disabled-checks:
- exitAfterDefer
forbidigo:
# Forbid the following identifiers
forbid:
- ^fmt\.Errorf$ # consider errors.Errorf in github.com/pkg/errors
linters:
# please, do not use `enable-all`: it's deprecated and will be removed soon.
# inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
fast: true
enable:
- bodyclose
- deadcode
- depguard
- dogsled
- dupl
- errcheck
- exportloopref
- exhaustive
#- funlen
#- goconst
- gocritic
#- gocyclo
- gofmt
- goimports
- goprintffuncname
#- gosec
- gosimple
- govet
- ineffassign
- misspell
- nolintlint
- rowserrcheck
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- unused
- varcheck
- whitespace
- prealloc
- predeclared
- asciicheck
- forbidigo
- makezero
- revive
#- interfacer
# don't enable:
# - scopelint
# - gochecknoglobals
# - gocognit
# - godot
# - godox
# - goerr113
# - interfacer
# - maligned
# - nestif
# - testpackage
# - wsl
run:
# default concurrency is a available CPU number.
# concurrency: 4 # explicitly omit this value to fully utilize available resources.
deadline: 5m
issues-exit-code: 1
tests: false
# output configuration options
output:
format: "colored-line-number"
print-issued-lines: true
print-linter-name: true
uniq-by-line: true
issues:
# Fix found issues (if it's supported by the linter)
fix: true
exclude-use-default: false
exclude:
- "Error return value of .((os.)?std(out|err)..*|.*Close|.*Flush|os.Remove(All)?|.*print(f|ln)?|os.(Un)?Setenv). is not check"

108
.goreleaser.yml Normal file
View File

@ -0,0 +1,108 @@
env:
- GO111MODULE=on
before:
hooks:
- go mod tidy
builds:
- id: nowin
env:
- CGO_ENABLED=0
- GO111MODULE=on
goos:
- linux
- darwin
goarch:
- 386
- amd64
- arm
- arm64
goarm:
- 7
ignore:
- goos: darwin
goarch: arm
- goos: darwin
goarch: 386
- goos: windows
goarch: arm
mod_timestamp: "{{ .CommitTimestamp }}"
flags:
- -trimpath
ldflags:
- -s -w -X main.VERSION=v{{.Version}}
- id: win
env:
- CGO_ENABLED=0
- GO111MODULE=on
goos:
- windows
goarch:
- 386
- amd64
goarm:
- 7
mod_timestamp: "{{ .CommitTimestamp }}"
flags:
- -trimpath
ldflags:
- -s -w -X main.VERSION=v{{.Version}}
- id: docker
env:
- CGO_ENABLED=0
- GO111MODULE=on
goos:
- linux
goarch:
- amd64
- arm64
- 386
- arm
mod_timestamp: "{{ .CommitTimestamp }}"
flags:
- -trimpath
ldflags:
- -s -w -X main.VERSION=v{{.Version}}
checksum:
name_template: "{{ .ProjectName }}_checksums.txt"
changelog:
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
- fix typo
- Merge pull request
- Merge branch
- Merge remote-tracking
- go mod tidy
archives:
- id: binary
builds:
- win
name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format_overrides:
- goos: windows
format: binary
- id: nowin
builds:
- nowin
- win
name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
format_overrides:
- goos: windows
format: zip
nfpms:
- license: AGPL 3.0
homepage: https://github.com/johlanse/study_xxqg
file_name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
formats:
- deb
- rpm
maintainer: johlanse
builds:
- nowin
- win

51
Dockerfile Normal file
View File

@ -0,0 +1,51 @@
FROM python:alpine
LABEL maintainer="${QL_MAINTAINER}"
ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/root/.local/share/pnpm/global/5/node_modules \
LANG=zh_CN.UTF-8 \
SHELL=/bin/bash \
PS1="\u@\h:\w \$ " \
QL_DIR=/ql \
QL_BRANCH=${QL_BRANCH}
WORKDIR ${QL_DIR}
RUN set -x \
&& sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories \
&& apk update -f \
&& apk upgrade \
&& apk --no-cache add -f bash \
coreutils \
moreutils \
git \
curl \
wget \
tzdata \
perl \
openssl \
nodejs \
jq \
openssh \
npm \
&& rm -rf /var/cache/apk/* \
&& apk update \
&& ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
&& echo "Asia/Shanghai" > /etc/timezone \
&& git config --global user.email "qinglong@@users.noreply.github.com" \
&& git config --global user.name "qinglong" \
&& git config --global http.postBuffer 524288000 \
&& npm install -g yarn \
&& rm -rf /root/.cache \
&& rm -rf /root/.npm \
COPY ./dist/docker_linux_$TARGETARCH*/qinglong-go ${QL_DIR}/qinglong-go
RUN chmod -R 777 ${QL_DIR}/qinglong-go
EXPOSE 8080
VOLUME ${QL_DIR}/data
CMD cd ${QL_DIR} && ./qinglong-go

View File

@ -8,6 +8,10 @@ import (
"path"
)
var (
VERSION = "UNKNOWN"
)
func Api(group *gin.RouterGroup) {
group.GET("", get())
}
@ -18,7 +22,7 @@ func get() gin.HandlerFunc {
exist := os.IsNotExist(err)
ctx.JSON(200, res.Ok(system.System{
IsInitialized: !exist,
Version: "2.0.14",
Version: VERSION,
LastCommitTime: "",
LastCommitId: "",
Branch: "master",

View File

View File

@ -16,6 +16,9 @@ import (
//go:embed config_sample.sh
var sample []byte
//go:embed package_sample.json
var pack []byte
func Api(group *gin.RouterGroup) {
group.GET("/", get())
group.PUT("/init", appInit())
@ -45,8 +48,11 @@ func appInit() gin.HandlerFunc {
_ = os.MkdirAll(path.Join("data", "log"), 0666)
_ = os.MkdirAll(path.Join("data", "repo"), 0666)
_ = os.MkdirAll(path.Join("data", "scripts"), 0666)
_ = os.MkdirAll(path.Join("data", "deps"), 0666)
_ = os.MkdirAll(path.Join("data", "raw"), 0666)
_ = os.WriteFile(path.Join("data", "config", "config.sh"), sample, 0666)
_ = os.WriteFile(path.Join("data", "config", "config_sample.sh"), sample, 0666)
_ = os.WriteFile(path.Join("data", "scripts", "package.json"), pack, 0666)
_ = os.WriteFile(path.Join("data", "config", "config.sample.sh"), sample, 0666)
type Req struct {
UserName string `json:"username"`
Password string `json:"password"`

16
docker-compose.yml Normal file
View File

@ -0,0 +1,16 @@
version: "3.5"
services:
xuexi-auto:
image: huoxue1/qinglong-go:latest
# 容器名
container_name: qinglong-go
environment:
# 时区
- TZ=Asia/Shanghai
# 配置文件路径
volumes:
- ./data:/ql/data
# 映射端口
ports:
- 8080:8080
restart: unless-stopped

View File

@ -3,6 +3,7 @@ package main
import (
nested "github.com/Lyrics-you/sail-logrus-formatter/sailor"
"github.com/huoxue1/qinglong-go/controller"
"github.com/huoxue1/qinglong-go/service"
rotates "github.com/lestrrat-go/file-rotatelogs"
log "github.com/sirupsen/logrus"
"io"
@ -12,7 +13,7 @@ import (
)
func init() {
w, err := rotates.New(path.Join("data", "logs", "%Y-%m-%d.log"), rotates.WithRotationTime(time.Hour*24))
w, err := rotates.New(path.Join("data", "log", "qinglong-go", "%Y-%m-%d.log"), rotates.WithRotationTime(time.Hour*24))
if err != nil {
log.Errorf("rotates init err: %v", err)
panic(err)
@ -36,6 +37,7 @@ func init() {
}
func main() {
service.AppInit()
engine := controller.Router()
_ = engine.Run(":8080")
}

View File

@ -48,6 +48,13 @@ func QueryRunningCron() ([]*Crontabs, error) {
return crontabs, err
}
func QueryCronByDir(dir string) ([]*Crontabs, error) {
crontabs := make([]*Crontabs, 0)
session := engine.Table(new(Crontabs)).Where(builder.Like{"command", "task " + dir + "%"})
err := session.Find(&crontabs)
return crontabs, err
}
func FindAllEnableCron() []*Crontabs {
crontabs := make([]*Crontabs, 0)
err := engine.Table(new(Crontabs)).Where("isdisabled=?", 0).Find(&crontabs)

29
service/app.go Normal file
View File

@ -0,0 +1,29 @@
package service
import (
"context"
"github.com/huoxue1/qinglong-go/utils"
log "github.com/sirupsen/logrus"
"os"
"path"
)
func AppInit() {
go runYarn()
}
func runYarn() {
defer func() {
recover()
}()
_, err := os.Stat(path.Join("data", "scripts", "package.json"))
if os.IsNotExist(err) {
return
}
ch := make(chan int, 1)
utils.RunTask(context.WithValue(context.Background(), "cancel", ch), "yarn install", map[string]string{}, func(ctx context.Context) {
log.Infoln("开始执行yarn初始化")
}, func(ctx context.Context) {
log.Infoln("yarn初始化执行完成")
}, os.Stdout)
}

View File

@ -129,16 +129,28 @@ func runCron(crontabs *models.Crontabs) {
}
func AddTask(crontabs *models.Crontabs) {
c := cron.New()
crons := strings.Split(crontabs.Schedule, " ")
var c *cron.Cron
if len(crons) == 5 {
c = cron.New()
} else if len(crons) == 6 {
c = cron.New(cron.WithParser(
cron.NewParser(cron.Second | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)))
} else {
log.Errorf("the task %s cron %s is error", crontabs.Name, crontabs.Command)
return
}
_, err := c.AddFunc(crontabs.Schedule, func() {
runCron(crontabs)
})
if err != nil {
log.Errorln("添加task错误" + err.Error())
log.Errorln("添加task错误" + crontabs.Schedule + err.Error())
return
}
c.Start()
manager.Store(crontabs.Id, c)
}
func handCommand(command string) *task {
@ -189,32 +201,3 @@ func handCommand(command string) *task {
}
return ta
}
//type myWriter struct {
// fileName string
//}
//
//func (m *myWriter) Write(p []byte) (n int, err error) {
// file, _ := os.OpenFile(m.fileName, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
// n, err = file.Write(p)
// file.Close()
// return n, err
//}
//
////通过管道同步获取日志的函数
//func syncLog(reader io.ReadCloser, writer io.Writer) {
// buf := make([]byte, 1)
// for {
// strNum, err := reader.Read(buf)
// if strNum > 0 {
// outputByte := buf[:strNum]
// writer.Write(outputByte)
// }
// if err != nil {
// //读到结尾
// if err == io.EOF || strings.Contains(err.Error(), "file already closed") {
// return
// }
// }
// }
//}

View File

@ -8,6 +8,7 @@ import (
"github.com/huoxue1/qinglong-go/service/config"
"github.com/huoxue1/qinglong-go/service/cron"
"github.com/huoxue1/qinglong-go/utils"
log "github.com/sirupsen/logrus"
"io"
"os"
"os/exec"
@ -45,12 +46,60 @@ func downloadFiles(subscriptions *models.Subscriptions) {
if err != nil {
return
}
if config.GetKey("AutoAddCron") == "true" {
addScripts(subscriptions)
} else {
log.Infoln("未配置自动添加定时任务,不添加任务!")
}
file, _ := os.OpenFile(subscriptions.LogPath, os.O_APPEND|os.O_RDWR, 0666)
file.WriteString(fmt.Sprintf("\n##执行结束.. %s耗时0秒\n\n", time.Now().Format("2006-01-02 15:04:05")))
_ = file.Close()
subscriptions.Status = 1
models.UpdateSubscription(subscriptions)
} else if subscriptions.Type == "file" {
addRawFiles(subscriptions)
}
}
func addRawFiles(subscriptions *models.Subscriptions) {
subscriptions.LogPath = "data/log/" + time.Now().Format("2006-01-02") + "/" + subscriptions.Alias + "_" + uuid.New().String() + ".log"
subscriptions.Status = 0
file, _ := os.OpenFile(subscriptions.LogPath, os.O_CREATE|os.O_RDWR, 0666)
defer file.Close()
_ = models.UpdateSubscription(subscriptions)
defer func() {
subscriptions.Status = 1
_ = models.UpdateSubscription(subscriptions)
}()
err := utils.DownloadFile(subscriptions.Url, path.Join("data", "raw", subscriptions.Alias))
if err != nil {
_, _ = file.WriteString(err.Error() + "\n")
return
}
name, c, err := getSubCron(path.Join("data", "raw", subscriptions.Alias))
if err != nil {
_, _ = file.WriteString(err.Error() + "\n")
return
}
utils.Copy(path.Join("data", "raw", subscriptions.Alias), path.Join("data", "scripts", subscriptions.Alias))
if c != "" {
command, err := models.GetCronByCommand(fmt.Sprintf("task %s", subscriptions.Alias))
if err != nil {
file.WriteString("已添加新的定时任务 " + name + "\n")
_, _ = cron.AddCron(&models.Crontabs{
Name: name,
Command: fmt.Sprintf("task %s", subscriptions.Alias),
Schedule: c,
Timestamp: time.Now().Format("Mon Jan 02 2006 15:04:05 MST"),
Status: 1,
Labels: []string{},
})
} else {
command.Name = name
command.Schedule = c
_ = cron.UpdateCron(command)
}
}
}
@ -98,6 +147,11 @@ func addScripts(subscriptions *models.Subscriptions) {
if err != nil {
return
}
crontabs, _ := models.QueryCronByDir(subscriptions.Alias)
cronMap := make(map[string]*models.Crontabs, len(crontabs))
for _, crontab := range crontabs {
cronMap[crontab.Command] = crontab
}
for _, entry := range dir {
// 判断文件后缀
if !utils.In(strings.TrimPrefix(filepath.Ext(entry.Name()), "."), extensions) {
@ -128,6 +182,7 @@ func addScripts(subscriptions *models.Subscriptions) {
command.Name = name
command.Schedule = c
_ = cron.UpdateCron(command)
delete(cronMap, command.Command)
}
}
@ -139,6 +194,12 @@ func addScripts(subscriptions *models.Subscriptions) {
}
}
}
if config.GetKey("AutoDelCron") == "true" {
for _, m := range cronMap {
file.WriteString("已删除失效的任务 " + m.Name + "\n")
models.DeleteCron(m.Id)
}
}
}
func getSubCron(filePath string) (name string, cron string, err error) {
@ -159,5 +220,6 @@ func getSubCron(filePath string) (name string, cron string, err error) {
} else {
return "", "", errors.New("not found cron")
}
cron = strings.TrimPrefix(cron, " ")
return
}

View File

@ -3,6 +3,7 @@ package utils
import (
log "github.com/sirupsen/logrus"
"io"
"net/http"
"os"
"path"
"path/filepath"
@ -48,3 +49,31 @@ func Copy(src, dest string) {
}
}
}
func DownloadFile(url, filePath string) error {
response, err := http.Get(url)
if err != nil {
return err
}
defer func(Body io.ReadCloser) {
err := Body.Close()
if err != nil {
}
}(response.Body)
file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
return err
}
defer func(file *os.File) {
err := file.Close()
if err != nil {
}
}(file)
_, err = io.Copy(file, response.Body)
if err != nil {
return err
}
return nil
}