1. 技术文章
  2. 使用bloom 加速sqler + gitbase 的代码统计分析情况

使用bloom 加速sqler + gitbase 的代码统计分析情况

由 僤鯓⒐⒋嵵緔 提交于 2020-04-19 15:49:06

我们基于gitbase 暴露的mysql 服务，可以方便的查询数据，但是如果需要长时间计算的就不太好了
还是我们可以通过bloom通过配置的方式就可以解决，以下是一个实践以及一些问题的解决访问

环境准备

  • docker-compose 文件
  
version: "3"
services: 
  lb: 
    image: openresty/openresty:alpine
    volumes:
    - "./nginx-lb.conf:/usr/local/openresty/nginx/conf/nginx.conf"
    ports:
    - "9000:80"
  bloom: 
    image: dalongrong/bloom:v1.28.0
    volumes: 
    - "./bloom.cfg:/etc/bloom.cfg"
    ports: 
    - "8811:8811"
    - "9001:8080"
  redis:
    image: redis
    ports:
    - "6379:6379"
  gitbase:
    container_name: gitbase
    hostname: gitbase
    image: srcd/gitbase:v0.24.0-rc2
    volumes: 
    - "./git-demos:/opt/repos"
    ports:
    - "3306:3306"
  sqler:
    image: dalongrong/sqler:2.2-pprof
    volumes:
    - "./sqler.hcl:/app/config.example.hcl"
    environment:
    - "DSN=root@tcp(gitbase:3306)/gitbase?multiStatements=true"
    ports:
    - "3678:3678"
    - "8025:8025"
  • 运行原理
    如下图：openresty进行api 的代理（自动添加header头以及proxy,bloom需要），bloom cache 请求api，sqler 暴露rest api
    gitbase 提供git sql 引擎服务

     

     




配置说明

  • openresty
  
worker_processes 1;
user root;  
events {
    worker_connections 1024;
}
http {
    include mime.types;
    default_type application/octet-stream;
    lua_need_request_body on;
    gzip on;
    resolver 127.0.0.11 ipv6=off;          
    real_ip_header X-Forwarded-For;
    real_ip_recursive on;
    server {
        listen 80;
        charset utf-8;
        default_type text/html;
        location / {
             default_type text/plain; 
             index index.html;
        }
        location /codecounts {
            proxy_pass http://bloom:8080;
            proxy_set_header Bloom-Request-Shard 0;
            proxy_set_header Host $host;
            proxy_read_timeout 10000;
            proxy_send_timeout 10000;
            proxy_buffer_size 1M; 
            proxy_buffers 8 1M; 
            proxy_busy_buffers_size 1M; 
            proxy_temp_file_write_size 1M;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host $server_name;
        }
        location /repos {
            proxy_pass <a href="http://bloom:8080;">http://bloom:8080;</a>
           // 核心，动态添加header
            proxy_set_header Bloom-Request-Shard 0;
            proxy_set_header Host $host;
            proxy_read_timeout 10000;
            proxy_send_timeout 10000;
            proxy_buffer_size 1M; 
            proxy_buffers 8 1M; 
            proxy_busy_buffers_size 1M; 
            proxy_temp_file_write_size 1M;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host $server_name;
        }
        location /apps {
            proxy_pass <a href="http://bloom:8080;">http://bloom:8080;</a>
           // 核心，动态添加header
            proxy_set_header Bloom-Request-Shard 0;
            proxy_set_header Host $host;
            proxy_read_timeout 10000;
            proxy_send_timeout 10000;
            proxy_buffer_size 1M; 
            proxy_buffers 8 1M; 
            proxy_busy_buffers_size 1M; 
            proxy_temp_file_write_size 1M;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Host $server_name;
        }
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
            root html;
        }
​
    }
}
  • bloom
    基于toml 的配置
  
[server]
​
log_level = "debug"
inet = "0.0.0.0:8080"
​
[control]
​
inet = "0.0.0.0:8811"
tcp_timeout = 900
​
[proxy]
​
[[proxy.shard]]
​
shard = 0
host = "sqler"
port = 8025
​
​
[cache]
​
ttl_default = 600
executor_pool = 64
disable_read = false
disable_write = false
compress_body = true
​
[redis]
host = "redis"
port = 6379
database = 0
pool_size = 80
max_lifetime_seconds = 60
idle_timeout_seconds = 600
connection_timeout_seconds = 1
​
max_key_size = 256000
max_key_expiration = 2592000
 
 
  • sqler 代码统计查询
codecounts {
    exec = <<SQL
 SET inmemory_joins = 1;
 SET SQL_SELECT_LIMIT=200;
SELECT
 repo,
 MONTH,
 YEAR,
 DAY,
 sum( JSON_EXTRACT( info, '$.Code.Additions' ) AS code_lines_added ) AS code_lines_addeds,
 sum( JSON_EXTRACT( info, '$.Code.Deletions' ) AS code_lines_removed ) AS code_lines_removeds 
FROM
 (
 SELECT
  repository_id AS repo,
  commit_stats ( repository_id, commit_hash ) AS info,
  commits.commit_author_when AS commit_when,
  YEAR ( committer_when ) AS YEAR,
  MONTH ( committer_when ) AS MONTH,
  DAY ( committer_when ) AS DAY 
 FROM
  ref_commits
  NATURAL JOIN commits 
 ) a 
GROUP BY
 repo,
 YEAR,
 MONTH,
    DAY
ORDER BY
 MONTH,
 YEAR,
 DAY
 limit 500;
SQL
}
​
cache {
    cron = "* 1 * * *"
    trigger {
        webhook = "http://lb/codecounts"
    }
}
​
apps {
   exec = <<SQL
   SET SQL_SELECT_LIMIT=200; 
      SET inmemory_joins = 1;
     select 1;
   SQL
}
​
repos {
exec = <<SQL
SET SQL_SELECT_LIMIT=500; 
SELECT
    repository_id,
    LANGUAGE(file_path, blob_content) as lang,
    SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Code')) as code,
    SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Comment')) as comments,
    SUM(JSON_EXTRACT(LOC(file_path, blob_content), '$.Blank')) as blanks,
    COUNT(1) as files
FROM refs
NATURAL JOIN commit_files
NATURAL JOIN blobs
WHERE ref_name='HEAD'
GROUP BY lang,repository_id;
SQL
}

几个问题

  • gzip 问题
    因为bloom是不处理数据解压缩的，但是sqler默认gzip了，所以使用中会有问题，解决方法，源码编译，禁用gzip部分
    参考
    server_rest.go 注意我同时添加了pprof，方便分析性能问题


  
// Copyright 2018 The SQLer Authors. All rights reserved.
// Use of this source code is governed by a Apache 2.0
// license that can be found in the LICENSE file.
package main
​
import (
 "net/http"
 _ "net/http/pprof"
 "strings"
​
 "github.com/labstack/echo"
 "github.com/labstack/echo/middleware"
)
​
// initialize RESTful server
func initRESTServer() error {
 e := echo.New()
 e.HideBanner = true
 e.HidePort = true
 e.Pre(middleware.RemoveTrailingSlash())
 e.Use(middleware.CORS())
 // e.Use(middleware.GzipWithConfig(middleware.GzipConfig{Level: 9}))
 e.Use(middleware.Recover())
 e.GET("/", routeIndex)
 e.Any("/:macro", routeExecMacro, middlewareAuthorize)
 e.GET("/debug/pprof/*", echo.WrapHandler(http.DefaultServeMux))
 return e.Start(*flagRESTListenAddr)
}
  • 大量代码走sqler查询很慢
    这个问题应该是gitbase 实现问题，默认用ide工具，测试很快，但是sqler就是很慢，发现如果我们执行了
    查询的数据（分页）就很快了，实际推荐还是走分页方式处理，如果数据是可估算的，可以通过
    SET SQL_SELECT_LIMIT=200; 解决


参考资料

https://github.com/rongfengliang/bloom-sqler-gitbase
https://github.com/alash3al/sqler
https://github.com/valeriansaliou/bloom

标签
github
openresty
Redis
nginx
git
MaxKey
iNet
易学教程内所有资源均来自网络或用户发布的内容，如有违反法律规定的内容欢迎反馈！
该文章没有解决你所遇到的问题?点击提问,说说你的问题,让更多的人一起探讨吧!