Python Golang Java DevOps Programmer

golang etcd watch

package etcd import ( "context" "encoding/json" "fmt" "LearnEtcd/tailf" "strings" "time" "github.com/astaxie/beego/logs" etcd_client "github.com/coreos/etcd/clientv3" "github.com/coreos/etcd/mvcc/mvccpb" "LearnEtcd/Ip" ) type EtcdClient struct { Client *etcd_client.Client Keys []string } var ( etcdClient *EtcdClient ) func InitEtcd(addr string, key string) (collectConf []tailf.CollectConf, err error) { cli, err := etcd_client.New(etcd_client.Config{ Endpoints: []string{"localhost:2379", "localhost:22379", "localhost:32379"}, DialTimeout: 5 * time.Second, }) if err != nil { logs.Error("connect etcd failed, err:", err) return } etcdClient = &EtcdClient{ Client: cli, } if strings. [Read More]

golang kafka

producer package kafka import ( "github.com/Shopify/sarama" "github.com/astaxie/beego/logs" ) var ( client sarama.SyncProducer ) func InitKafka(addr string) (err error) { config := sarama.NewConfig() config.Producer.RequiredAcks = sarama.WaitForAll config.Producer.Partitioner = sarama.NewRandomPartitioner config.Producer.Return.Successes = true client, err = sarama.NewSyncProducer([]string{addr}, config) if err != nil { logs.Error("init kafka producer failed, err:", err) return } logs.Debug("init kafka succ") return } func SendToKafka(data, topic string) (err error) { msg := &sarama.ProducerMessage{} msg.Topic = topic msg.Value = sarama. [Read More]

AriaNg+Caddy FileManager打造自己的离线下载网盘

简单介绍 Aria2 是一个命令行下运行、多协议、多来源下载工具,支持磁力链接、BT 种子、HTTP、FTP 等下载协议,当然因为它是命令行下载工具,所以我们想下载一个东西还需要去敲命令自然是不方便,于是就有一些人根据 Aria2 的 API 开发了一些在线管理面板,可以直接在网页上面添加管理任务。 本教程介绍的是 AriaNg,Github 项目:https://github.com/mayswind/AriaNg Aria2 是一个后端 (服务端) 的程序,而 AriaNg 是一个前端 (HTML+JS 静态) 控制面板,不需要和 Aria2(后端 / 服务端)放在一个服务器或者设备中,你甚至可以只在服务器上面搭建 Aria2 后端,然后访问别人建好的 AriaNg 前端面板,也可以远程操作 Aria2 后端! centos6 安装Aria2 wget http://repository.it4i.cz/mirrors/repoforge/redhat/el6/en/x86_64/rpmforge/RPMS/rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm rpm -ivh rpmforge-release-0.5.3-1.el6.rf.x86_64.rpm yum -y install aria2 # 测试 aria2c -v 配置后端 # 创建Aria2配置文件夹 mkdir /root/.aria2 && cd /root/.aria2 # 下载Aria2配置文件、DHT文件(BT有没有速度就靠这个) wget --no-check-certificate https://softs.fun/Other/Aria2/aria2.conf wget --no-check-certificate https://softs.fun/Other/Aria2/dht.dat # 创建一个空的 Aria2 下载任务保存文件 echo '' > /root/. [Read More]

caddy nginx 对比

Caddy 是什么? Caddy 是一个多功能的 HTTP web服务器,并且使用Let’s Encrypt提供的免费证书,自动让网站升级到HTTPS Every Site on HTTPS Caddy is a general-purpose HTTP/2 web server that serves HTTPS by default. Fork Caddy On Github 为什么要使用 Caddy 安全 Caddy 是一个默认使用https协议的web服务器 无依赖 Caddy 使用 Go 语言编写,编译好的二进制文件能够运行在任何支持Go语言的平台,不需要自己安装任何库。 使用简单 Caddy 的配置简单,不管你是新的web开发者,还是专业人士,都能够快速上手 Caddy 一键安装脚本 wget -N --no-check-certificate https://raw.githubusercontent.com/ToyoDAdoubi/doubi/master/caddy_install.sh && chmod +x caddy_install.sh && bash caddy_install.sh 这里我们看一下官网的例子说明 Caddy Documentation :2015 # Host: (any), Port: 2015 localhost # Host: localhost; Port: 2015 localhost:8080 # Host: localhost; Port: 8080 example. [Read More]

golang sync.map

package main

import (
	"fmt"
	"sync"
)

func main() {
	list := map[string]interface{}{
		"name":          "田馥甄",
		"birthday":      "1983年3月30日",
		"age":           34,
		"hobby":         []string{"听音乐", "看电影", "电视", "和姐妹一起讨论私人话题"},
		"constellation": "白羊座",
	}

	var m sync.Map
	for k, v := range list {
		m.Store(k, v)
	}

	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		m.Store("age", 22)
		m.LoadOrStore("tag", 8888)
		wg.Done()
	}()

	go func() {
		m.Delete("constellation")
		m.Store("age", 18)
		wg.Done()
	}()

	wg.Wait()

	m.Range(func(key, value interface{}) bool {
		fmt.Println(key, value)
		return true
	})
}

使用 rclone 将文件同步至 Google Dirve

首先是下载文件 yum install unzip wget -y wget -O https://downloads.rclone.org/rclone-current-linux-amd64.zip unzip rclone-current-linux-amd64.zip cd rclone-*-linux-amd64 然后复制文件到相关路径 cp rclone /usr/bin/ chown root:root /usr/bin/rclone chmod 755 /usr/bin/rclone 新建一个配置 rclone config n) New remote d) Delete remote q) Quit config e/n/d/q> n name> google(你的配置名称,此处随意填写但之后需要用到) Type of storage to configure. Choose a number from below, or type in your own value 1 / Amazon Drive \ "amazon cloud drive" 2 / Amazon S3 (also Dreamhost, Ceph, Minio) \ "s3" 3 / Backblaze B2 \ "b2" 4 / Dropbox \ "dropbox" 5 / Encrypt/Decrypt a remote \ "crypt" 6 / Google Cloud Storage (this is not Google Drive) \ "google cloud storage" 7 / Google Drive \ "drive" 8 / Hubic \ "hubic" 9 / Local Disk \ "local" 10 / Microsoft OneDrive \ "onedrive" 11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) \ "swift" 12 / SSH/SFTP Connection \ "sftp" 13 / Yandex Disk \ "yandex" Storage> 7(根据网盘类型选择Google Dirve) Google Application Client Id - leave blank normally. [Read More]

宝塔linux 控制面板

Centos安装命令:

yum install -y wget && wget -O install.sh http://download.bt.cn/install/install.sh && sh install.sh

Ubuntu/Deepin安装命令:

wget -O install.sh http://download.bt.cn/install/install-ubuntu.sh && sudo bash install.sh

重启

service bt restart

spark dataframe

from pyspark.sql import SparkSession from pyspark.sql import Row spark = SparkSession.builder.appName('test').getOrCreate() sc = spark.sparkContext spark.conf.set("spark.sql.shuffle.partitions", 6) l = [('Ankit',25),('Jalfaizy',22),('saurabh',20),('Bala',26)] rdd = sc.parallelize(l) people = rdd.map(lambda x: Row(name=x[0], age=int(x[1]))) schemaPeople = spark.createDataFrame(people) df = spark.read.format("csv"). \ option("header", "true") \ .load("iris.csv") df.printSchema() df.show(10) df.count() df.columns df.withColumn('newWidth',df.SepalWidth * 2).show() df.drop('Name').show() df.describe().show() df.describe('Name').show() #分类变量 df.select('Name','SepalLength').show() df.select('Name').distinct().count() ### 分组统计 groupby(colname).agg({'col':'fun','col2':'fun2'}) df.groupby('Name').agg({'SepalWidth':'mean','SepalLength':'max'}).show() ### 自定义的汇总方法 import pyspark.sql.functions as fn df.agg(fn.count('SepalWidth').alias('width_count'), fn.countDistinct('id').alias('distinct_id_count')).collect() ### 数据集拆成两部分 trainDF, testDF = df.randomSplit([0.6, 0. [Read More]