最近读了一本老外写的电子书《Network Programming with Go》,感觉写得还可以,在这里将书中一些重点记录一下以备忘。
跟其它书不同,这本书的第一章主要讲了分布式系统与传统单机系统在架构层面的区别。其中有4节我觉得还挺重要的,设计分布式系统时可以多参考这些方面。
###Points of Failure###
Distributed applications run in a complex environment. This makes them much more prone to failure than standalone applications on a single computer. The points of failure include
Applications have to be designed with these possible failures in mind. Any action performed by one component must be recoverable if failure occurs in some other part of the system. Techniques such as transactions and continuous error checking need to be employed to avoid errors.
###Acceptance Factors###
###Transparency###
The “holy grails” of distributed systems are to provide the following:
###Eight fallacies of distributed computing###
Sun Microsystems was a company that performed much of the early work in distributed systems, and even had a mantra “The network is the computer.” Based on their experience over many years a number of the scientists at Sun came up with the following list of fallacies commonly assumed:
Many of these directly impact on network programming. For example, the design of most remote procedure call systems is based on the premise that the network is reliable so that a remote procedure call will behave in the same way as a local call. The fallacies of zero latency and infinite bandwidth also lead to assumptions about the time duration of an RPC call being the same as a local call, whereas they are magnitudes of order slower.
The recognition of these fallacies led Java’s RMI (remote method invocation) model to require every RPC call to potentially throw a RemoteException. This forced programmers to at least recognise the possibility of network error and to remind them that they could not expect the same speeds as local calls.
第三章就讲到golang里面的socket编程了。
type IP []byte # IP类型
ip := net.ParseIP(ipStr) # 由string解析出IP类型
type IPMask []byte # IPMask类型
func IPv4Mask(a, b, c, d byte) IPMask # 得到一个IPMask
func (ip IP) DefaultMask() IPMask # 得到一个IP地址默认的IPMask
func (ip IP) Mask(mask IPMask) IP # 得到一个IP地址所对应的网络地址
type IPAddr {
IP IP
} # IPAddr类型
func ResolveIPAddr(net, addr string) (*IPAddr, os.Error) # 由string解析出一个IPAddr, 如net.ResolveIPAddr("ip4", "192.168.1.1")
func LookupHost(name string) (cname string, addrs []string, err os.Error) # 根据主机名查找其对应的CName与IP地址
func LookupPort(network, service string) (port int, err os.Error) # 查询一个服务的默认端口, 如port, err := net.LookupPort("tcp", "telnet")
type TCPAddr struct {
IP IP
Port int
} # TCPAddr类型
func ResolveTCPAddr(net, addr string) (*TCPAddr, os.Error) # 解析出一个TCPAddr类型, 如addr, err := net.ResolveTCPAddr("tcp", "192.168.1.1:23")
# TCPConn实现了Writer与Reader接口
func (c *TCPConn) Write(b []byte) (n int, err os.Error)
func (c *TCPConn) Read(b []byte) (n int, err os.Error)
# 客户端连接TCP,如conn, err := net.DialTCP("tcp", nil, addr)
func DialTCP(net string, laddr, raddr *TCPAddr) (c *TCPConn, err os.Error)
# 服务端监听TCP,如listener, err := net.ListenTCP("tcp", addr)
func ListenTCP(net string, laddr *TCPAddr) (l *TCPListener, err os.Error)
# 服务端接受了一个连接,如conn, err := listener.Accept()
func (l *TCPListener) Accept() (c Conn, err os.Error)
TCP服务端的一般范式为
service := ":1201"
tcpAddr, err := net.ResolveTCPAddr("tcp", service)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
for {
conn, err := listener.Accept()
if err != nil {
continue
}
// run as a goroutine
go handleClient(conn)
}
# 解析出UDP的地址
func ResolveUDPAddr(net, addr string) (*UDPAddr, os.Error)
# 客户端连接UDP, 如conn, err := net.DialUDP("udp", nil, addr)
func DialUDP(net string, laddr, raddr *UDPAddr) (c *UDPConn, err os.Error)
# 服务端监听UDP,如conn, err := net.ListenUDP("udp", addr)
func ListenUDP(net string, laddr *UDPAddr) (c *UDPConn, err os.Error)
# 服务端读写UDP包数据
func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err os.Error
func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (n int, err os.Error)
UDP服务端的一般范式为
service := ":1200"
udpAddr, err := net.ResolveUDPAddr("udp", service)
checkError(err)
conn, err := net.ListenUDP("udp", udpAddr)
checkError(err)
for {
handleClient(conn)
}
# 客户端连接Raw Socket
func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error)
# 服务端监听Raw Socket
func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error)
# 客户端或服务端读写Raw Socket
func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error)
func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error)
更范型化的接口
# 客户端连接服务端,如conn, err := net.Dial("tcp", nil, addr) 或 conn, err := net.Dial("udp", nil, addr) 或 conn, err := net.Dial("ip", nil, addr)
func Dial(net, laddr, raddr string) (c Conn, err os.Error)
# 服务端监听TCP,如listener, err := net.Listen("tcp", addr)
func Listen(net, laddr string) (l Listener, err os.Error)
# 服务端接受TCP连接, 如conn, err := listener.Accept()
func (l Listener) Accept() (c Conn, err os.Error)
# 服务端监听UDP, 如conn, err := net.ListenPacket("udp", addr)
func ListenPacket(net, laddr string) (c PacketConn, err os.Error)
# 服务端读写UDP包数据
type PacketConn interface {
ReadFrom(b []byte) (n int, addr Addr, err error)
WriteTo(b []byte, addr Addr) (n int, err error)
}
The Go net package recommends using these interface types rather than the concrete ones. But by using them, you lose specific methods such as SetKeepAlive or TCPConn and SetReadBuffer of UDPConn, unless you do a type cast. It is your choice.
第四章讲的是数据序列化方案
# 将对象marshal为字节数组
func Marshal(val interface{}) ([]byte, error)
# 从字节数组unmarshal回对象
func Unmarshal(b []byte, val interface{}) (rest []byte, err error)
这个主要用于读写X.509证书。
# 获得一个json encoder
encoder := json.NewEncoder(writer)
# 将对象序列化
err := encoder.Encode(obj)
encoder.Close()
# 获得一个json decoder
decoder := json.NewDecoder(reader)
# 将对象反序列化
err := decoder.Decode(&obj)
json序列化方案功能比较完善,语言互操作性好,序列化后结果也可读便于查错。但其基于文本,性能与大小可能没有基于字节流的好。
# 获得一个gob encoder
encoder := gob.NewEncoder(writer)
# 将对象序列化
err := encoder.Encode(obj)
encoder.Close()
# 获得一个gob decoder
decoder := gob.NewDecoder(reader)
# 将对象反序列化
err := decoder.Decode(&obj)
golang语言特有的序列化方案,功能完善,高效,但语言互操作性不好。
base64序列化方案
# 获得一个base64 encoder
encoder := base64.NewEncoder(base64.StdEncoding, writer)
# 将对象序列化
err := encoder.Encode(obj)
encoder.Close()
# 直接进行base64序列化
str := base64.StdEncoding.EncodeToString(obj)
# 获得一个base64 decoder
decoder := base64.NewDecoder(base64.StdEncoding, reader)
# 将对象反序列化
err := decoder.Decode(&obj)
# 直接进行base64反序列化
bb, err := base64.StdEncoding.DecodeString(str)
可以看到golang里各种编码器的API很类似,用会一个,其它就举一反三了,这点很好。
第5章主要讲应用层如何设计。其中讲到的概念其实以前做网络编程都涉及过,只不过在这章归纳总结后,更清晰了。
应用层协议设计主要有以下方面需要注意:
详细的说明可参阅Application-Level Protocols
上述这些问题并没有一个确定性的答案,需要根据具体场景作决策。
这里针对两种不同的数据格式,服务端的代码范式如下
# 基于字节流的数据格式
handleClient(conn) {
while (true) {
byte b = conn.readByte()
switch (b) {
case MSG_TYPE_1: ...
case MSG_TYPE_2: ...
...
}
}
}
# 基于文本的数据格式
handleClient(conn) {
while (true) {
line = conn.readLine()
if (line.startsWith(...) {
...
} else if (line.startsWith(...) {
...
}
}
}
第6章主要讲了字符集与文字编码问题,这里有一个概念要理解一下。
golang内部使用UTF-8字符编码对字符串进行编码,UTF-8字符编码是一种针对Unicode的可变长度字符编码。因此在编程中会用到以下方法。
str := "百度一下,你就知道"
# 得到字符串进行UTF-8编码后最后字节数组的长度
println("Byte length", len(str))
# 得到字符串中字符的个数
println("String length", len([]rune(str)))
字符集与字符编码的问题很重要,但事实上平时在编程中遇到比较多的可能仅仅是读写中文字符文件,这个记录一下,其它编码的处理也类似。
fileName := "test.txt"
if _, err := os.Stat(fileName); os.IsNotExist(err) {
_, err := os.Create(fileName)
checkErr(err)
}
f, err := os.Open(fileName)
checkErr(err)
writer := transform.NewWriter(f, simplifiedchinese.GB18030.NewEncoder())
io.WriteString(writer, "百度一下,你就知道")
writer.Close()
reader := transform.NewReader(f, simplifiedchinese.GB18030.NewDecoder())
bb, err := ioutil.ReadAll(reader)
checkErr(err)
fmt.Println(string(bb))
f.Close()
第7章主要讲安全,在编程中主要用到的有以下几种技术。
# 下面的md5的使用,其它sha1, sha256等hash算法的使用方法类似
hash := md5.New() // hash := md5.NewMD5([]byte{27, 23, 13, 55})
bytes := []byte("hello\n")
hash.Write(bytes)
hashValue := hash.Sum(nil)
# 这里演示了blowfish对称加密算法,其它AES,DES使用方法类似
key := []byte("my key")
cipher, err := blowfish.NewCipher(key)
if err != nil {
fmt.Println(err.Error())
}
# 加密
enc := make([]byte, 255)
cipher.Encrypt(enc, []byte("hello world"))
# 解密
decrypt := make([]byte, 8)
cipher.Decrypt(decrypt, enc)
fmt.Println(string(decrypt))
这里仅说明了下如何生成一对RSA公私钥及如何加载RSA公私钥,如何生成证书及如何加载证书,因为在编程中很少自己进行非对称加密,一般用在TLS连接会话里。而使用方法多数仅仅只是在建立连接时配置一下证书。
# 生成一对公私钥,并保存到文件
func main() {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
checkErr(err)
publicKey := &(privateKey.PublicKey)
savePEMKey("pri.key", privateKey)
savePEMPublicKey("pub.key", publicKey)
}
func savePEMPublicKey(path string, key *rsa.PublicKey) {
file, err := os.Create(path)
checkErr(err)
defer file.Close()
publicKey, err := ssh.NewPublicKey(key)
checkErr(err)
file.Write(ssh.MarshalAuthorizedKey(publicKey))
}
func savePEMKey(path string, key *rsa.PrivateKey) {
file, err := os.Create(path)
checkErr(err)
defer file.Close()
block := &pem.Block{
Type : "RSA PRIVATE KEY",
Bytes : x509.MarshalPKCS1PrivateKey(key),
}
pem.Encode(file, block)
}
func checkErr(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v", err)
os.Exit(-1)
}
}
# 从文件中加载公私钥
func main() {
privateKey := loadPEMKey("pri.key")
fmt.Printf("%v\n", *privateKey)
publicKey := loadPEMPublicKey("pub.key")
fmt.Printf("%v\n", *publicKey)
}
func loadPEMKey(path string) *rsa.PrivateKey {
file, err := os.Open(path)
checkErr(err)
defer file.Close()
bb, err := ioutil.ReadAll(file)
block, _ := pem.Decode(bb)
privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
checkErr(err)
return privateKey
}
func loadPEMPublicKey(path string) *rsa.PublicKey{
file, err := os.Open(path)
checkErr(err)
defer file.Close()
bb, err := ioutil.ReadAll(file)
checkErr(err)
pkey, _, _, _, err:= ssh.ParseAuthorizedKey(bb)
checkErr(err)
if pkey, ok := pkey.(ssh.CryptoPublicKey); ok {
publicKey := pkey.CryptoPublicKey().(*rsa.PublicKey)
return publicKey
}
return nil
}
func checkErr(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v", err)
os.Exit(-1)
}
}
# 生成证书,并保存证书到文件
func main() {
generateCert("test.cer.pem")
}
func generateCert(path string) {
random := rand.Reader
privateKey := loadPEMKey("pri.key")
now := time.Now()
then := now.Add(60 * 60 * 24 * 365 * 1000 * 1000 * 1000) // one year
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: "jan.newmarch.name",
Organization: []string{"Jan Newmarch"},
},
NotBefore: now,
NotAfter: then,
SubjectKeyId: []byte{1, 2, 3, 4},
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
BasicConstraintsValid: true,
IsCA: true,
DNSNames: []string{"jan.newmarch.name", "localhost"},
}
derBytes, err := x509.CreateCertificate(random, &template,
&template, &(privateKey.PublicKey), privateKey)
checkErr(err)
block := &pem.Block{
Type: "CERTIFICATE",
Bytes: derBytes,
}
certCerFile, err := os.Create(path)
checkErr(err)
pem.Encode(certCerFile, block)
certCerFile.Close()
}
func loadPEMKey(path string) *rsa.PrivateKey {
file, err := os.Open(path)
checkErr(err)
defer file.Close()
bb, err := ioutil.ReadAll(file)
block, _ := pem.Decode(bb)
privateKey, _ := x509.ParsePKCS1PrivateKey(block.Bytes)
return privateKey
}
func checkErr(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v", err)
os.Exit(-1)
}
}
# 从文件中加载证书
func main() {
cert := loadCert("test.cer.pem")
fmt.Printf("%v\n", *cert)
}
func loadCert(path string) *x509.Certificate {
certCerFile, err := os.Open(path)
checkErr(err)
bb, err := ioutil.ReadAll(certCerFile)
checkErr(err)
certCerFile.Close()
block, _ := pem.Decode(bb)
// trim the bytes to actual length in call
cert, err := x509.ParseCertificate(block.Bytes)
checkErr(err)
return cert;
}
func checkErr(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "error: %v", err)
os.Exit(-1)
}
}
# TCP服务端启用TLS
func main() {
startTCPServerWithTLS()
}
func startTCPServerWithTLS() {
cert, err := tls.LoadX509KeyPair("test.cer.pem", "pri.key")
checkErr(err)
config := tls.Config{Certificates: []tls.Certificate{cert}}
now := time.Now()
config.Time = func() time.Time { return now }
config.Rand = rand.Reader
_, err = tls.Listen("tcp", ":1200", &config)
checkErr(err)
...
}
# TCP客户端启用TLS
func main() {
startTCPClientWithTLS()
}
func startTCPClientWithTLS() {
conn, err := tls.Dial("tcp", "127.0.0.1:1200", nil)
checkErr(err)
handleClient(conn)
}