2022-01-23  17858276a536d625d9f99955c69a5edf 99+   20 分钟   3.1 k  0次访问

视频系统架构搭建文档

视频系统架构搭建文档

工作中部署视频电影架构。写个文档记录下。按照最低部署，后期流量上去了对应扩容就行

架构需要准备5台服务器，我用的微软云服务器 分别是

1台网关服务器，微软内网ip 172.19.16.10

1台部署Jar包服务器，微软内网ip 172.19.0.10

1台数据库服务器，微软内网ip 172.19.0.14

1台视频存储服务器 线下机房公网ip 142.4.10.65

1台图片存储服务器。 微软内网ip 172.19.16.15

如果需要开启app下载功能。还得部署一台视频下载服务器。

架构工作流程，APP视频软件通过app访问域名(网关服务器)，进入到，==》Jar服务器，通过Jar包接口配置的数据库接口进行查询。包括对应的视频播放地址，和图片存放地址。

相关对应的域名: 网关域名 txdy78.tv 播放域名 fybofang.com 图片域名 txtupian.com 视频分享域名 txfx.tv

视频系统后台管理域名 txadminav.com

1. 部署网关服务器

首先基础的linux相关环境搭建好。我这里主要写下nginx相关配置文档

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
#我设置的配置文件目录 ls查看
/software/站点配置文件
db  gateway-common  gateway.conf  share.conf  ssl

cat gateway.conf 
limit_req_zone $binary_remote_addr zone=mylimit:100m rate=1r/s;
# 后端jar服务器地址接口
upstream bg-admin1 {
    server 172.19.0.10:8081;
    #server 10.170.15.220:8081; 后期需要加负载直接添加就行
}
upstream bg-app1 {
    server 172.19.0.10:8082;
    #server 10.170.15.220:8082;
}

server {
    listen 80;
    server_name txdy78.tv www.txdy78.tv;

    # 具体代理配置
    include /software/站点配置文件/gateway-common/common.conf;
}

server {
    listen 888 ssl http2;
    server_name txdy78.tv www.txdy78.tv;

    # SSL
    ssl_certificate /software/域名证书/txdy78.tv/txdy78.tv_chain.crt;
    ssl_certificate_key /software/域名证书/txdy78.tv/txdy78.tv_key.key;

    # security
    include /software/站点配置文件/ssl/ssl.conf;

    # 具体代理配置
    include /software/站点配置文件/gateway-common/common.conf;
}

gateway-common/common.conf 配置内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
 cat gateway-common/common.conf 
# . files
location ~ /\.(?!well-known) {
    deny all;
}

# favicon.ico
location = /favicon.ico {
    log_not_found off;
    access_log off;
}

# restrict methods
if ($request_method !~ ^(GET|POST|PUT|PATCH|DELETE|HEAD|CONNECT|OPTIONS|TRACE)$) {
    return '405';
}

# CORS
location / {
    add_header Access-Control-Allow-Origin '*';
    add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS';
    add_header Access-Control-Allow-Headers 'content-type,token,version,DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization';

    if ($request_method = 'OPTIONS') {
        return 204;
    }
}

location =/ {
    default_type text/plain;
    echo "ok";
}

# 后台管理
location /videomanager {
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_connect_timeout 300;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_pass http://bg-admin1;
    proxy_next_upstream error timeout http_500 http_502 http_503 http_504 non_idempotent;
}

# 支付接口服务器
location /videosite/pay/ {
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_connect_timeout 300;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_pass http://43.154.58.46:8082; #支付接口服务器
}

# 限流
location /videosite/video/get/play-info {
    limit_req zone=mylimit burst=5 nodelay;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_connect_timeout 300;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_pass http://bg-app1;
    proxy_next_upstream error timeout http_500 http_502 http_503 http_504 non_idempotent;
}

# APP后台
location /videosite {
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_pass http://bg-app1;
    proxy_connect_timeout 300;
    proxy_http_version 1.1;
    proxy_set_header Connection "";
    proxy_next_upstream error timeout http_500 http_502 http_503 http_504 non_idempotent;
}

ssl/ssl.conf配置内容

里面的/etc/nginx/conf/dhparam.pem 文件直接本地通过命令生成即可

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
openssl dhparam -out /etc/nginx/conf/dhparam.pem 2048   #生成dhparam.pem文件

cat ssl/ssl.conf 
# SSL
ssl_session_timeout    1d;
ssl_session_cache      shared:SSL:10m;
ssl_session_tickets    off;

# Diffie-Hellman parameter for DHE ciphersuites
ssl_dhparam            /etc/nginx/conf/dhparam.pem;

# Mozilla Intermediate configuration
ssl_protocols          TLSv1.2 TLSv1.3;
ssl_ciphers            ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;

# OCSP Stapling
ssl_stapling           on;
ssl_stapling_verify    on;
resolver               1.1.1.1 1.0.0.1 8.8.8.8 8.8.4.4 208.67.222.222 208.67.220.220 valid=60s;
resolver_timeout       2s;

# security headers
add_header X-XSS-Protection          "1; mode=block" always;
add_header X-Content-Type-Options    "nosniff" always;
add_header Referrer-Policy           "no-referrer-when-downgrade" always;
add_header Content-Security-Policy   "default-src 'self' http: https: data: blob: 'unsafe-inline'; frame-ancestors 'self';" always;
add_header Permissions-Policy        "interest-cohort=()" always;
# HSTS
#add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always;

# . files
location ~ /\.(?!well-known) {
    deny all;
}

db/db.conf配置内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
stream {
    # Logging
    log_format main '[$time_local] $protocol $remote_addr '
    '$status $bytes_sent $bytes_received '
    '$session_time "$upstream_addr" "$upstream_bytes_sent" '
    '"$upstream_bytes_received" "$upstream_connect_time"';

    access_log logs/tcp-access.log main;
    error_log logs/tcp-error.log;

    upstream mysql {
        server 172.19.0.14:3368 weight=1;
    }

    upstream redis {
        server 172.19.0.14:6380 weight=1;
    }

    server {
        listen 3368;

        proxy_connect_timeout 3s;
        proxy_pass mysql;
    }

    server {
        listen 6380;

        proxy_connect_timeout 3s;
        proxy_pass redis;
    }
}

share.conf 分享域名配置文件内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 cat share.conf 
server {
    listen 80;
    server_name txfx.tv www.txfx.tv;

    location / {
        return 301 https://txfx.tv:88$request_uri;
    }
}

server {
    listen 443 ssl http2;
    listen 88 ssl http2;
    server_name txfx.tv www.txfx.tv;

    root    /software/share;
    default_type    text/html;

    # SSL
    ssl_certificate /software/域名证书/txfx.tv/txfx.tv_chain.crt;
    ssl_certificate_key /software/域名证书/txfx.tv/txfx.tv_key.key;

    # security
    include /software/站点配置文件/ssl/ssl.conf;

    # index.html fallback
    location / {
        try_files $uri $uri/ /proc.html?u=$uri;
    }

    location /redirect {
        content_by_lua_block {
            if not ngx.var.arg_u then
                ngx.redirect("/");
            end

            local http = require "resty.http";
            local httpc = http.new();
            local res, err = httpc:request_uri("http://txdy78.tv/videosite/short-link/getShortLink2/"..ngx.var.arg_u);

            if not res then
                ngx.log(ngx.ERR, "not res:" .. err);
                ngx.exit(ngx.ERROR);
            end

            if 200 ~= res.status then
                ngx.exit(res.status);
            end

            local json = require("cjson.safe");
            local t = json.decode(res.body);

            if not t or t.data == json.null or not t.data.longs then
                ngx.exit(404);
            end

            ngx.redirect("https://tx888.tv/?"..t.data.longs);
        }
    }
}

分享家目录内容 /software/share

1
2
3
4
ls /software/share/
index.html  proc.html  qq.html  wx.html
里面的具体内容请参考我另外的文章
https://yc6.cool/2021/07/31/nginx%E5%88%A4%E6%96%AD%E5%BE%AE%E4%BF%A1%E7%AB%AF%E8%B7%B3%E8%BD%AC%E4%B8%8D%E5%90%8C%E9%A1%B5%E9%9D%A2/

2. 部署Jar服务器。

这台服务器首先需要部署docker环境以及docker-compose

基础搭建环境我这里就不详细写文档了。主要写下部署文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
vim docker-compose.yml 

version: "3"
services:
  site:
    build: ./video_site
    container_name: video_site
    tty: true
    restart: always
    network_mode: "host"
    volumes:
      - /etc/localtime:/etc/localtime
      - /var/log/video-box:/var/log/video-box
    logging:
        driver: "json-file"
        options:
            max-file: "20"
            max-size: "10G"
  manager:
    build: ./video_manager
    container_name: video_manager
    tty: true
    restart: always
    network_mode: "host"
    volumes:
      - /etc/localtime:/etc/localtime
      - /var/log/video-box:/var/log/video-box
    logging:
        driver: "json-file"
        options:
            max-file: "20"
            max-size: "10G"

部署运行命令

1
2
docker-compose up -d --build #运行
docker-compose down   #停止

视频系统的后台管理也是部署这台。配文件内容是

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
 cat manager.conf 
# txadminav.com
server {
    listen                               443 ssl http2;
    server_name                          txadminav.com;
    root                                 /software/admin;

    # SSL
    ssl_certificate                      /software/域名证书/txadminav.com/txadminav.com_chain.crt;
    ssl_certificate_key                  /software/域名证书/txadminav.com/txadminav.com_key.key;

    # security headers
    add_header X-XSS-Protection          "1; mode=block" always;
    add_header X-Content-Type-Options    "nosniff" always;
    add_header Referrer-Policy           "no-referrer-when-downgrade" always;
    add_header Content-Security-Policy   "default-src 'self' http: https: data: blob: 'unsafe-inline'; frame-ancestors 'self';" always;
    add_header Permissions-Policy        "interest-cohort=()" always;
    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;

    # . files
    location ~ /\.(?!well-known) {
        deny all;
    }

    # index.html fallback
    location / {
        try_files $uri $uri/ /index.html;
    }

    # reverse proxy
    location /apis/ {
        proxy_pass                         http://www.txdy78.tv/;
        proxy_http_version                 1.1;
        proxy_cache_bypass                 $http_upgrade;

        # Proxy headers
        proxy_set_header Upgrade           $http_upgrade;
        proxy_set_header Connection        $connection_upgrade;
        proxy_set_header Host              $host;
        proxy_set_header X-Real-IP         $remote_addr;
        proxy_set_header Forwarded         $proxy_add_forwarded;
        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header X-Forwarded-Host  $host;
        proxy_set_header X-Forwarded-Port  $server_port;

        # Proxy timeouts
        proxy_connect_timeout              60s;
        proxy_send_timeout                 60s;
        proxy_read_timeout                 120s;
    }

    # favicon.ico
    location = /favicon.ico {
        log_not_found off;
        access_log    off;
    }

    # robots.txt
    location = /robots.txt {
        log_not_found off;
        access_log    off;
    }

    # assets, media
    location ~* \.(?:css(\.map)?|js(\.map)?|jpe?g|png|gif|ico|cur|heic|webp|tiff?|mp3|m4a|aac|ogg|midi?|wav|mp4|mov|webm|mpe?g|avi|ogv|flv|wmv)$ {
        expires    7d;
        access_log off;
    }

    # svg, fonts
    location ~* \.(?:svgz?|ttf|ttc|otf|eot|woff2?)$ {
        add_header Access-Control-Allow-Origin "*";
        expires    7d;
        access_log off;
    }
}

# HTTP redirect
server {
    listen      80;
    server_name txadminav.com;
    return      301 https://txadminav.com$request_uri;
}

3. 数据库服务器部署

首先也是搭建好docker环境，redis数据库通过docker部署就行了

1
2

docker run --name redis   --restart=unless-stopped   -v /etc/localtime:/etc/localtime:ro   -v redis-data:/data   -v redis-conf:/etc/redis   -p 6380:6379   -d redis:6.2.6   redis-server --appendonly yes   --requirepass "wuji..!@#redis..#@!abc"

mysql数据直接在服务器搭建就好了。具体搭建文档请参考我之前的文章

1
2
#文章地址
https://yc6.cool/2020/08/03/mysql%E5%9F%BA%E7%A1%80/

4. 部署视频存储系统和图片存储系统

部署这两台服务器。我都是通过fdfs文件系统部署的。

1
2
具体部署的详细步骤，参考我以前的文章
https://yc6.cool/2020/08/04/FastdFS%E9%83%A8%E7%BD%B2%E8%AF%A6%E8%A7%A3/

下面是更新后的fdfs部署文档

FastDFS部署

一、安装依赖

1
yum install git gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl-devel wget vim ncurses-devel libevent-devel -y

二、安装FastDFS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
mkdir /data/fdfs/{storage,client,tracker}/{data,logs} -p

mkdir /fdfs

cd /fdfs

git clone https://github.com/happyfish100/fastdfs

git clone https://github.com/happyfish100/libfastcommon

cd /fdfs/libfastcommon 

./make.sh && ./make.sh install

cd /fdfs/fastdfs

./make.sh && ./make.sh install

./setup.sh /etc/fdfs/

systemctl daemon-reload

1、配置Tracker

  1. 编辑配置文件

vim /etc/fdfs/tracker.conf

1
2
3
base_path=/home/yuqing/fastdfs # 基础路径  修改为/data/fdfs/tracker
reserved_storage_space = 20% # storage保留空间 修改为0.1%
use_storage_id = false # 指定storage 改为true
  1. 修改storage_ids.conf

vim /etc/fdfs/storage_ids.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# <id>  <group_name>  <ip_or_hostname[:port]>
#
# id is a natural number (1, 2, 3 etc.),
# 6 bits of the id length is enough, such as 100001
#
# storage ip or hostname can be dual IPs seperated by comma,
# one is an inner (intranet) IP and another is an outer (extranet) IP,
# or two different types of inner (intranet) IPs
# for example: 192.168.2.100,122.244.141.46
# another eg.: 192.168.1.10,172.17.4.21
#
# the port is optional. if you run more than one storaged instances
# in a server, you must specified the port to distinguish different instances.

100001   group1  116.202.174.78 # 修改为指定storage的ip
100002   group1  192.168.0.197
  1. 配置开机自启

systemctl enable fdfs_trackerd

  1. 启动trakcer

systemctl start fdfs_trackerd

  1. 查看tracker日志

tailf /data/fdfs/tracker/logs/trackerd.log

  1. 配置openresty
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
mkdir /software/站点配置文件/ -p
cat <<EOF >/software/站点配置文件/fdfs.conf
upstream fdfs_group1 {
    # 修改为对应storage
    server 47.56.185.141:8888 weight=1 max_fails=2 fail_timeout=30s;
}
server {
    listen 80;
    server_name 47.75.70.29; # 修改为对应值
    add_header Access-Control-Allow-Origin *;
    add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS';
    add_header Access-Control-Allow-Headers 'content-type,token,version,DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization';
    if (\$request_method = 'OPTIONS') {
        return 204;
    }
    location /group1/M00 {
        proxy_next_upstream http_502 http_504 error timeout invalid_header;
        proxy_pass http://fdfs_group1;
        expires 30d;
    }
}
EOF
openresty -s reload

2、配置Storage

  1. 编辑配置文件

vim /etc/fdfs/storage.conf

主要配置以下几项

1
2
3
4
5
group_name=group1 # 组名
base_path=/home/yuqing/fastdfs # 基础路径  修改为/data/fdfs/storage
store_path0=/home/yuqing/fastdfs # 存储路径 修改为/data/fdfs/storage
tracker_server=192.168.209.121:22122 # tracker地址 修改为对应地址
tracker_server=192.168.209.122:22122 # 若单tracker 可注释
  1. 编译安装openresty及fastdfs-nginx-module
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
cd /fdfs

git clone https://github.com/happyfish100/fastdfs-nginx-module

git clone https://github.com/google/ngx_brotli

cd ngx_brotli && git submodule update --init

cd /fdfs

wget https://openresty.org/download/openresty-1.19.9.1.tar.gz

tar xf openresty-1.19.9.1.tar.gz

cd openresty-1.19.9.1/

./configure --prefix=/usr/local/openresty --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-http_ssl_module --with-compat --add-module=/fdfs/fastdfs-nginx-module/src --add-module=/fdfs/ngx_brotli

gmake && gmake install

systemctl restart openresty
  1. 配置开机自启

systemctl enable fdfs_storaged

  1. 启动storage

systemctl start fdfs_storaged

  1. 查看storage日志

tailf /data/fdfs/storage/logs/storaged.log

  1. 配置mod_fastdfs.conf

cp /fdfs/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/

vim /etc/fdfs/mod_fastdfs.conf

主要配置以下几项

1
2
3
4
5
tracker_server=tracker:22122 # tracker地址 修改为对应地址
group_name=group1 # 组名
url_have_group_name = false # url是否包含组名 修改为true
store_path0=/home/yuqing/fastdfs # storage存储路径 修改为/data/fdfs/storage
response_mode=proxy # 同步未完成时响应模式 修改为redirect
  1. 修改mime.types
1
2
video/mp2t                                      ts
application/vnd.apple.mpegurl                   m3u8
  1. 配置openresty
1
2
3
4
5
6
7
8
9
10
11
mkdir /software/站点配置文件/ -p
cat <<EOF >/software/站点配置文件/fdfs.conf
server {
    listen 8888;
    server_name 47.56.185.141; # 此处进行对应修改
    location /group1/M00 {
        ngx_fastdfs_module;
    }
}
EOF
openresty -s reload

3、配置client

vim /etc/fdfs/client.conf

主要配置以下几项

1
2
3
base_path=/home/yuqing/fastdfs # 基础路径  修改为/data/fdfs/client
tracker_server=192.168.209.121:22122 # tracker地址 修改为对应地址
tracker_server=192.168.209.122:22122 # 若单tracker 可注释

三、测试

1、查看FastDFS集群状态

fdfs_monitor /etc/fdfs/client.conf

2、上传文件测试

1
2
[root@Tracker ~]$ fdfs_upload_file /etc/fdfs/client.conf /fdfs/fastdfs/README_zh.md 
group1/M00/00/00/Lzi5jV3hBsGAB4RaAAAGpOD9zYI2054.md

若返回 fid 则上传成功

3、文件访问测试

使用浏览器访问http://<storage ip>:8888/group1/M00/00/00/Lzi5jV3hBsGAB4RaAAAGpOD9zYI2054.md若能正常下载，即集群正常工作。

视频存储服务器搭建好后交给专门上传视频的人员进行上传。

喜欢这篇文章？打赏一下作者吧

评论

关注我

:D 一言句子获取中...

最新评论

加载中，最新评论有1分钟缓存...

最新文章

分类

归档

标签

linux18
docker16
Prometheus14
html14
python12
web12
nginx9
mysql7
容器7
数据库7
科学上网6
work5
自动化5
博客4
工作4
工具4
ssl3
文件系统3
日志3
日志加密3
查看全部>>

订阅更新

输入邮箱开始订阅，更博后邮件通知！