Golang tool:include spider library,image library and some other db library such as mysql,redis,mogodb,hbase,cassandra
2024-08-26 14:20:02
一.Go_tool
This is a tool library for Golang.Dont't worry about not understant it!
All comment writes by English,Ahaha~~
Oh,I think some will be Chinese.
二.Usage
go get -u -v github.com/hunterhug/go_tool
go get -v github.com/hunterhug/go_image
go get -v github.com/hunterhug/go-hbase
go get -v gopkg.in/redis.v4
go get -v github.com/gocql/gocql
go get -v golang.org/x/net/context
中国防火某些库无法下载!!!
三.Include
1.Image 图像处理库 (image deal library)
package image
import (
"testing"
)
func TestImage(t *testing.T) {
// Scale a image file by cuting 100*100
err := ThumbnailF2F("../data/image.png", "../data/image100-100.png", 100, 100)
if err != nil {
t.Error("Test ThumbnailF2F:" + err.Error())
}
// Scale a image file by cuting width:200 (Equal scaling)
err = ScaleF2F("../data/image.png", "../data/image200.png", 200)
if err != nil {
t.Error("Test ScaleF2F:" + err.Error())
}
// File Real name
filename, err := RealImageName("../data/image.png")
if err != nil {
t.Error("Test RealImageName:" + err.Error())
} else {
t.Log("Test RealImageName::real filename" + filename)
}
}
2.Spider 爬虫封装库 (spider library)
package spider
import (
"testing"
"fmt"
)
func TestSpider(t *testing.T) {
// global log record
//SetLogLevel("DEBUg")
SetLogLevel("debug")
// GLOBAL TIMEOUT
SetGlobalTimeout(3)
// a new spider without proxy
// NewSpider(nil)
proxy := "http://smart:smart2016@104.128.121.46:808"
spiders,err := NewSpider(proxy)
if err!=nil{
panic(err)
}
// method can be get and post
spiders.Method = "get"
// wait times,can zero
spiders.Wait = 2
// which url fetch
spiders.Url = "http://www.goole.com"
// a new header,default ua, no refer
spiders.NewHeader(nil, "www.google.com", nil)
// go!fetch url --||
body, err := spiders.Go()
if err != nil {
Log().Error(err)
} else {
// bytes get!
fmt.Printf("%s", string(body))
}
Log().Debugf("%#v",spiders)
// if filesize small than 500KB
err = TooSortSizes(body, 500)
Log().Error(err.Error())
}
3.Dbs 数据库封装库 (database library)
Mysql
package mysql
import (
"testing"
)
/*
CREATE TABLE IF NOT EXISTS `51job_keyword` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`keyword` varchar(255) NOT NULL DEFAULT '',
`address` varchar(255) NOT NULL DEFAULT '',
`kind` varchar(255) NOT NULL DEFAULT '',
`created` datetime DEFAULT NULL,
`updated` datetime DEFAULT NULL,
`time51` int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=utf8 COMMENT='关键字表';
*/
func TestMysql(t *testing.T) {
// mysql config
config := MysqlConfig{
Username: "root",
Password: "smart2016",
Ip: "127.0.0.1",
Port: "3306",
Dbname: "51job",
}
// a new db connection
db := New(config)
// open connection
db.Open()
// create sql
sql := `
CREATE TABLE IF NOT EXISTS 51job.51job_keyword (
id int(11) NOT NULL AUTO_INCREMENT,
keyword varchar(255) NOT NULL DEFAULT '',
address varchar(255) NOT NULL DEFAULT '',
kind varchar(255) NOT NULL DEFAULT '',
created datetime DEFAULT NULL,
updated datetime DEFAULT NULL,
time51 int(11) NOT NULL DEFAULT '0',
PRIMARY KEY (id)
) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=utf8 COMMENT='关键字表';`
// create
inum, err := db.Create(sql)
if err != nil {
t.Error(err.Error())
} else {
t.Logf("create number:%d\n", inum)
}
// insert sql
//'1', '教师', '潮州', '0', '2016-05-27 00:00:00', '2016-05-27 00:00:00', '204'
sql = "INSERT INTO `51job_keyword`(`keyword`,`address`,`kind`) values(?,?,?)"
// insert
num, err := db.Insert(sql, "PHP", "潮州", 0)
if err != nil {
t.Error(err.Error())
} else {
t.Logf("insert number:%d\n", num)
}
// select sql
sql = "SELECT * FROM 51job_keyword where address=? and kind=? limit ?;"
// select
result, err := db.Select(sql, "潮州", 0, 6)
if err != nil {
t.Error(err.Error())
} else {
// values
for row, v := range result {
t.Logf("%v:%#v\n", row, v)
}
}
}
Redis
package myredis
import (
"testing"
)
func TestRedis(t *testing.T) {
config := RedisConfig{}
config.DB = 0
config.Host = "127.0.0.1:6379"
config.Password = "smart2016" // no password set ""
client, err := NewRedis(config) // new redis client
if err != nil {
panic(err)
}
// set key==value
err = client.Set("key", "value", 0)
if err != nil {
t.Error(err.Error())
}
// get key
val, err := client.Get("key")
if err != nil {
panic(err)
} else {
t.Log("Redis value:" + val)
}
// push test,pust pool with b value
num, err := client.Lpush("pool", "b")
if err != nil {
t.Error(err.Error())
}
// total length of list
t.Log(num)
// pushx test,will be error if not exist pool10
num, err = client.Lpushx("pool10", "b")
if err != nil {
t.Error(err.Error())
}
t.Log(num)
// len test
t.Log(client.Llen("pool"))
// pop test
pops, e := client.Lpop("pool2")
if e!=nil {
t.Logf("%v,%v", pops, e)
}
// bpop test
bpops, e := client.Blpop(0,"pool1","pool1")
t.Logf("%#v,%v", bpops, e)
//rpoplpush test POOL1 empty so will be redis.nil
rpoplpush,e:=client.Rpoplpush("POOL1","pool1")
t.Logf("%#v,%v", rpoplpush, e)
//brpoplpush test POOL1 empty so will be redis.nil if timeout but zero set is wait a long time
brpoplpush,e:=client.Brpoplpush("POOL1","pool1",15)
t.Logf("%#v,%v", brpoplpush, e)
}
Cassandra
package cassandra
import (
"github.com/gocql/gocql"
"testing"
)
/*
测试之前先填充一下cassandra语句
Before test:
create keyspace example with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };
create table example.tweet(timeline text, id UUID, text text, PRIMARY KEY(id));
create index on example.tweet(timeline);
*/
func TestCdb(t *testing.T) {
//先构造一个字符串数组连接
// cassandra host
host := []string{"192.168.11.74"}
//指定cassandra keyspace,类似于mysql中的db
// cassandra keyspace like mysql db
keyword := "example"
//连接
// connect a cdb
cdb := NewCdb(host, keyword)
//构造插入语句
// insert sql!
insertsql := cdb.Query(`INSERT INTO tweet (timeline, id, text) VALUES (?, ?, ?)`,
"me", gocql.TimeUUID(), "hello world")
//执行,查看Exec方法,可知执行后已经关闭
// exec a insert operation
if err := insertsql.Exec(); err != nil {
t.Fatal(err)
}
//构造查找语句
// query sql
querysql := cdb.Query(`SELECT "id", text FROM "tweet"`)
//执行
// done it !
iter := querysql.Iter()
//定义字节数组
// id uuid in cassandra
var id gocql.UUID
var text string
//循环取值,需要手工,无法再封装
// take value just like that i can't wrap it again, and no need
for iter.Scan(&id, &text) {
t.Logf("Tweet:%v,%v\n", id, text)
}
//这个需要关闭
// should be close
if err := iter.Close(); err != nil {
t.Logf("%v", err)
}
}
Hbase
package myhbase
import (
"github.com/hunterhug/go-hbase"
"testing"
"fmt"
"time"
)
/*
before test,To do bellow
create_namespace 'hunterhug'
create 'hunterhug:info','colfamily',{SPLITALGO => 'HexStringSplit',NUMREGIONS => 40}
*/
func TestHbase(t1 *testing.T) {
var (
//格式化时间
// Time format
fs = "2006-01-02 15:04:05"
//hbase配置
//hbase config
config = HbaseConfig{
Zkport: "2181",//zk
Zkquorum: "192.168.11.73",
}
//创建一个客户端
// new client
clients = New(config)
)
//打开客户端
//open connection
clients.Open()
//放主键
//set a rowkey
rowkey := "rowkey"
//命名空间
//set namespace
hbasenamespace := "hunterhug"
//表名
//set tablename
hbasetable := "info"
//列族
//set colfamily
hbasefamily := "colfamily"
//set col
hbasecol := "go"
hbasecol1 := "die"
//放值,第一列
// put a value to hunterhug:info colfamily:go
put := hbase.CreateNewPut([]byte(rowkey))
put.AddStringValue(hbasefamily, hbasecol, "value")
_, err := clients.Client.Put(hbasenamespace + ":" + hbasetable, put)
if err != nil {
t1.Logf(err.Error())
}
//第二列
// the same,put another value
put.AddStringValue(hbasefamily, hbasecol1, "value1")
_, err = clients.Client.Put(hbasenamespace + ":" + hbasetable, put)
if err != nil {
t1.Logf(err.Error())
}
//获取一行
//get value by rowkey
result, err := clients.GetResult(hbasenamespace + ":" + hbasetable, rowkey)
if err != nil {
t1.Logf(err.Error())
}
if v, exist := result.Columns[hbasefamily + ":" + hbasecol]; exist {
fmt.Printf("found row:%v,v:%v\n",rowkey,v.Value.String())
}
//扫描
// create a scan object,so scan can be close because client is inside in it
scan := clients.Client.Scan(hbasenamespace + ":" + hbasetable)
//过滤两列,只拿
// just scan one col
err = scan.AddString(hbasefamily + ":" + hbasecol)
if err != nil {
panic(err)
}
// no!we want two col
err = scan.AddString(hbasefamily + ":" + hbasecol1)
if err != nil {
panic(err)
}
//过滤时间戳,只拿时间戳在这段范围的合数据
// and we are just want before 720h's data
var t = time.Now().Add(-1 * 720 * time.Hour * 1)
fmt.Printf("Get Data which record before %v\n", t.Format(fs))
// Time filter
scan.SetTimeRangeFrom(t)
var v1, v2 string
//循环处理值
// get scan value
scan.Map(func(r *hbase.ResultRow) {
// if value exist,v1!
if v, exist := r.Columns[hbasefamily + ":" + hbasecol]; exist {
v1 = v.Value.String()
}
if v, exist := r.Columns[hbasefamily + ":" + hbasecol1]; exist {
v2 = v.Value.String()
}
// output
fmt.Printf("row:%s\t %s:%s \t %s:%s \t", r.Row.String(), hbasecol, v1, hbasecol1, v2)
})
}
4.Log 日志库 (log library)
package mylog
import (
"testing"
)
func TestLog(t *testing.T) {
// log filename
logger:=NewLog("test.log")
// log level
logger.SetLevel("error")
logger.Println("1.println","11.println")
logger.Error("error")
logger.Debugf("%v","debug")
}
5.Util 文件/时间等杂项库 (some small library such as times and file)
package util
import (
"testing"
)
func TestUtil(t *testing.T) {
// test int to string
i := 2
if "2" == IS(i) {
t.Log("Test IS:int to string")
}
// test string to int
v, err := SI("e2")
if err == nil && v == i {
t.Log("Test SI:string to int")
} else {
t.Log("Test SI:" + err.Error())
}
// caller dir
t.Log("Test CurDir:"+CurDir())
// create dir
err = MakeDir("../data")
if err != nil {
t.Log("Test MakeDir:" + err.Error())
} else {
t.Log("Test MakeDir:dir already exist")
}
// create dir by filename
filename := "../data/testutil.txt"
err = MakeDirByFile(filename)
if err != nil {
t.Log("Test MakeDirByFile:" + err.Error())
} else {
t.Log("Test MakeDirByFile: dir already exist")
}
// save bytes into file
err = SaveToFile(filename, []byte("testutil"))
if err != nil {
t.Log("Test SaveToFile" + err.Error())
}
// read bytes from file
filebytes, err := ReadfromFile(filename)
if err != nil {
t.Error("Test ReadfromFile:" + err.Error())
} else {
t.Log("Test ReadfromFile:" + string(filebytes))
}
// times format
t.Log(TodayString(3))
// file exist?
t.Logf("%v",FileExist("../r.txt"))
// find the go file in some dir
filenames,err:=ListDir(`G:\smartdogo\src\github.com\hunterhug\go_tool`,"go")
t.Logf("%v:%v",filenames,err)
// devide a string list into severy string list
stringlist:=[]string{"2","3","4","5","4"}
num:=3
result,err:=DevideStringList(stringlist,num)
if err!=nil{
t.Error(err)
}else{
t.Logf("%#v",result)
}
// now secord times from January 1, 1970 UTC.
secord:=GetSecordTimes()
t.Log(secord)
// now date string by secord
timestring:=GetSecord2DateTimes(secord)
t.Log(timestring)
// change back
t.Log(GetDateTimes2Secord(timestring))
}
四.How to use
You all can read the test golang file.And I recomment use IDE pycharm which python language use,
can also install The Go plugin.
五.Author
一只尼玛
My website:http://www.lenggirl.com
My Twitter:https://twitter.com/hunterhug_
My Weibo:http://weibo.com/hunterhug
Updating...
最新文章
- 读书笔记--SQL必知必会--常用MySQL(MariaDB)命令
- 常用js归纳
- Java上等价类划分测试的实现
- IOS 关键字self,super,copy, retain, assign , readonly , readwrite, nonatomic、 @synthesize、@property、@dynamic
- GPS开发之知识储备(NMEA0183)
- core
- 解决修改重命名sql Server数据库名称
- Android 监听键盘的弹起与收缩
- JBOss启动只能在本机访问的解决办法
- 关于MySQL
- Makefile 快速入门
- mlock家族:锁定物理内存
- 深度学习“引擎”之争:GPU加速还是专属神经网络芯片?
- hyperv虚拟机网络速度慢问题的解决办法
- 1.自定义控制器切换<;一>;
- javaweb:推断当前请求是否为移动设备訪问
- AccessibilityService+WindowManager+SurfaceView开系统权限
- linux常用命令小结
- wb 黑名单批量操作
- 简单了解weblogic配置文件