python numpy pandas

def fun1():
    def fun2():
        def fun3():
            print("Hello world!")
        return fun3
    return fun2

a=fun1()
# a()
fun1()()()

Hello world!

def fun1(x):
    def fun2(y):
        print(x+y)
    return fun2
fun1(2)(3)

5

def addTips(fun):
    def wrap(*arg,**kwargs):
        print("this is oper before")
        result = fun(*arg,**kwargs)
        #return result
        print("this is oper after")
        return result
    return wrap
@addTips
def add():
    print("进行添加的操作")
    
add()

this is oper before
进行添加的操作
this is oper after

@addTips
def add1(x,y):
    print(x+y)
add1(2,6)

this is oper before
8
this is oper after

@addTips
def add2(x,y):
    return x+y

print(add2(2,3))

this is oper before
this is oper after
5

def addTips(i):
    def wrap1(fun):
        def wrap(*arg,**kwargs):
            print("this is oper before")
            result = 0
            if i > 10:
                result = fun(*arg,**kwargs)
            else:
                print("you not have any auth")
            #return result
            print("this is oper after")
            return result
        return wrap
    return wrap1

@addTips(11)
def add(x,y):
    print(x+y)
    
add(2,3)

this is oper before
5
this is oper after

import itertools #引入迭代工具

x = range(1,5)
#排列
com1 = itertools.combinations(x,3)
for i in com1:
    print(i)

(1, 2, 3)
(1, 2, 4)
(1, 3, 4)
(2, 3, 4)

#组合
x = range(1,3)
com2 = itertools.permutations(x,2)
for i in com2:
    print(i)

(1, 2)
(2, 1)

#笛卡尔积
x = range(1,3)
y = list("abc")
com3 = itertools.product(x,y)
for i in com3:
    print(i)

(1, 'a')
(1, 'b')
(1, 'c')
(2, 'a')
(2, 'b')
(2, 'c')

#串联迭代器
com4 = itertools.chain(com1,com2,com3)
for i in com4:
    print(i)

print([i**2 for i in range(8) if i%2==0])

[0, 4, 16, 36]

a = [[1,2,3],[44,55,66],[77,88,99]]
print([j for i in a for j in ib

[1, 2, 3, 44, 55, 66, 77, 88, 99]

from  functools  import reduce 
lambda1 = lambda x: x**2
lambda2 = lambda x,y:x+y
lambda3 = lambda x:x%2==0
#python map reduce filter
print(list(map(lambda1,range(8))))
print(reduce(lambda2,range(8)))
print(list(filter(lambda3,range(8))))

[0, 1, 4, 9, 16, 25, 36, 49]
28
[0, 2, 4, 6]

#使用map和reduce函数
#创建一个相乘的函数
mult = lambda x,y:x*y
jiecheng = lambda n:reduce(mult,range(1,n+1))
sumResult = lambda x:x+y
print(list(map(jiecheng,range(1,6))))
a = list(map(jiecheng,range(1,6)))
sumResult = lambda x,y :x+y
result = reduce(sumResult,a)
print(result)

[1, 2, 6, 24, 120]
153

#pymysql   mysqldb  sqlalchemy
import pymysql
# 1.创建数据库连接
conn = pymysql.connect(host="127.0.0.1",user='root',password='root',db='jobdb',charset='utf8')
# 2.创建操作游标
cursor = conn.cursor()
# 3.创建字符编码与自动提交
cursor.execute('set names utf8')
cursor.execute('set autocommit=1')
# 4.编写sql语句与crud
cursor.execute('show tables')
sql = 'select * from jobdata limit 5'
print(sql)
# 5.执行sql获取结果
huangshu= cursor.execute(sql)
print(huangshu)
# 得到结果集的3种方式  fetchone fetchall fetchmany
result = cursor.fetchall()
print(result)
# 5.关闭游标和资源
cursor.close()
conn.close()

select * from jobdata limit 5
5
((1, 'Java', 'JAVA开发', '至真信息招聘', '3k-5k', '广州', '经验应届毕业生', '本科及以上', '全职', 'https://www.lagou.com/jobs/2574027.html'), (2, 'Java', 'JAVA工程师', '车主无忧研发中心招聘', '8k-15k', '广州', '经验1-3年', '本科及以上', '全职', 'https://www.lagou.com/jobs/2738861.html'), (3, 'Java', 'JAVA开发工程师', '卡宝宝技术部招聘', '7k-14k', '广州', '经验1-3年', '本科及以上', '全职', 'https://www.lagou.com/jobs/2983700.html'), (4, 'Java', 'JAVA开发工程师', '奥威亚开发部招聘', '10k-15k', '广州', '经验1-3年', '本科及以上', '全职', 'https://www.lagou.com/jobs/2979903.html'), (5, 'Java', 'Java', '橙医生技术部招聘', '8k-15k', '广州', '经验1-3年', '大专及以上', '全职', 'https://www.lagou.com/jobs/1049383.html'))

import time
#时间戳
print(time.time())
#当前时间,打印出时间原组
print(time.localtime())
# 2017-12-21 15:12:34  %I 12时进制
print(time.strftime('%Y-%m-%d %H:%M:%S',time.localtime()))
print(time.strptime('18-08-05 15:12:34','%y-%m-%d %H:%M:%S'))

1535507407.372698
time.struct_time(tm_year=2018, tm_mon=8, tm_mday=29, tm_hour=9, tm_min=50, tm_sec=7, tm_wday=2, tm_yday=241, tm_isdst=0)
2018-08-29 09:50:07
time.struct_time(tm_year=2018, tm_mon=8, tm_mday=5, tm_hour=15, tm_min=12, tm_sec=34, tm_wday=6, tm_yday=217, tm_isdst=-1)

import numpy as np
# array zero  empty arange linspace  random
a = np.array([[2,3,4],[3,4,6]],dtype=np.float)
print(a)

[[ 2.  3.  4.]
 [ 3.  4.  6.]]

b = np.zeros([2,3],dtype=np.int)
print(b)
c = np.ones([2,3])
print(c)
d = np.empty((2,2))
print(d)

[[0 0 0]
 [0 0 0]]
[[ 1.  1.  1.]
 [ 1.  1.  1.]]
[[  1.27422810e-311   2.33419537e-312]
 [  8.48798317e-313   8.70018275e-313]]

print(np.arange(1,10,2))

[1 3 5 7 9]

#等差数列
e = np.linspace(1,10,7,endpoint=False)
print(e)

[ 1.          2.28571429  3.57142857  4.85714286  6.14285714  7.42857143
  8.71428571]

#等比数列
f = np.logspace(1,8,10,dtype=np.int)
print(f)

[       10        59       359      2154     12915     77426    464158
   2782559  16681005 100000000]

# random
g = np.random.random((2,3,4))
print(g)

[[[ 0.25702055  0.98376051  0.05585929  0.6229259 ]
  [ 0.43491759  0.82742601  0.71256847  0.85558227]
  [ 0.52091075  0.40166471  0.6514136   0.07676163]]

 [[ 0.35497533  0.24796663  0.299326    0.88137736]
  [ 0.69668304  0.25432368  0.05165381  0.04155755]
  [ 0.36588213  0.73055171  0.98575158  0.35841199]]]

print(np.random.randint(1,10,5))
print(g.imag)

[4 2 5 7 9]
[[[ 0.  0.  0.  0.]
  [ 0.  0.  0.  0.]
  [ 0.  0.  0.  0.]]

 [[ 0.  0.  0.  0.]
  [ 0.  0.  0.  0.]
  [ 0.  0.  0.  0.]]]

a = np.array([2900,3,4])
b = a.astype(np.float)
print(b)

[ 2900.     3.     4.]

d = np.random.random((2,3,5))
print(d,d.shape)

[[[ 0.43498115  0.10203122  0.49607491  0.89564589  0.45151256]
  [ 0.09605607  0.55805833  0.62551507  0.02656035  0.92505993]
  [ 0.06856241  0.74115976  0.70165392  0.38507679  0.97801944]]

 [[ 0.02604068  0.61297118  0.21805418  0.89961424  0.90764166]
  [ 0.5773425   0.31270289  0.40884587  0.12665826  0.21801203]
  [ 0.27060303  0.91790138  0.32900214  0.42332602  0.71642873]]] (2, 3, 5)

e = d.reshape((3,10))
print(e,e.shape)

[[ 0.43498115  0.10203122  0.49607491  0.89564589  0.45151256  0.09605607
   0.55805833  0.62551507  0.02656035  0.92505993]
 [ 0.06856241  0.74115976  0.70165392  0.38507679  0.97801944  0.02604068
   0.61297118  0.21805418  0.89961424  0.90764166]
 [ 0.5773425   0.31270289  0.40884587  0.12665826  0.21801203  0.27060303
   0.91790138  0.32900214  0.42332602  0.71642873]] (3, 10)

#数组和列表之间的运算是不同的，数组之间的运算是对应位置之间的运算
a = np.array([[1,2,3],[9,6,3]])
b = np.array([[4,5,6],[8,5,2]])
print(a+b)
print(a-b)
print(a*b)
print(a/b)
print(b.shape)

[[ 5  7  9]
 [17 11  5]]
[[-3 -3 -3]
 [ 1  1  1]]
[[ 4 10 18]
 [72 30  6]]
[[ 0.25   0.4    0.5  ]
 [ 1.125  1.2    1.5  ]]
(2, 3)

d = b.reshape((3,2))
#点乘等操作
c = a.dot(d)
print(c)

[[31 27]
 [87 99]]

import numpy as np
a = np.random.random((2,3,4))
print(a)
print(a[0][1][1])
print(a[0,:,1:3])
print(a[1,:])
#切片的获得是一个视图，在上面操作就相当于在原数组上进行操作

[[[ 0.28211282  0.48742378  0.26441143  0.22300644]
  [ 0.33328369  0.46119033  0.33145709  0.97908699]
  [ 0.78985878  0.0059506   0.78379064  0.77909717]]

 [[ 0.62321309  0.41451579  0.99233426  0.80932283]
  [ 0.08101114  0.25087679  0.61401797  0.25568332]
  [ 0.90907157  0.91881851  0.83182079  0.00798843]]]
0.461190326442
[[ 0.48742378  0.26441143]
 [ 0.46119033  0.33145709]
 [ 0.0059506   0.78379064]]
[[ 0.62321309  0.41451579  0.99233426  0.80932283]
 [ 0.08101114  0.25087679  0.61401797  0.25568332]
 [ 0.90907157  0.91881851  0.83182079  0.00798843]]

#布尔索引
a = np.random.random((4,4,2))
print(a)
c = a > 0.5
print(c)
d = a[c]
print(d)

[[[  4.44650177e-01   4.84549948e-01]
  [  1.63097853e-01   6.95758696e-01]
  [  7.74627929e-01   4.02019958e-01]
  [  3.92688349e-01   8.47190342e-02]]

 [[  6.50061373e-01   8.67824711e-01]
  [  5.22127742e-01   4.45195482e-01]
  [  9.07419468e-01   3.82062991e-01]
  [  9.62987734e-01   4.04880803e-01]]

 [[  4.70068137e-01   8.64142833e-01]
  [  4.25149078e-02   3.89552038e-01]
  [  1.61686526e-01   9.76093351e-01]
  [  1.27423596e-01   2.76400473e-01]]

 [[  7.27683444e-01   5.72721041e-01]
  [  2.27198847e-04   6.36574348e-01]
  [  7.75000651e-01   7.31438620e-02]
  [  1.93906550e-01   6.48449010e-01]]]
[[[False False]
  [False  True]
  [ True False]
  [False False]]

 [[ True  True]
  [ True False]
  [ True False]
  [ True False]]

 [[False  True]
  [False False]
  [False  True]
  [False False]]

 [[ True  True]
  [False  True]
  [ True False]
  [False  True]]]
[ 0.6957587   0.77462793  0.65006137  0.86782471  0.52212774  0.90741947
  0.96298773  0.86414283  0.97609335  0.72768344  0.57272104  0.63657435
  0.77500065  0.64844901]

#花式索引
a = np.arange(32).reshape(8,-1)
print(a)
print(a[1:3])
#非连续的行
print(a[[0,3,5]])
#非连续的行与列,这种取出的是0,0  3,3  5,2三个元素
print(a[[0,3,5],[0,3,2]])
#使用索引器取出元素,这种取出的是0,0 0,2, 0,3 3,0 3,2  3,3  ......
print(a[np.ix_([0,3,5],[0,2,3])])

[[ 0  1  2  3]
 [ 4  5  6  7]
 [ 8  9 10 11]
 [12 13 14 15]
 [16 17 18 19]
 [20 21 22 23]
 [24 25 26 27]
 [28 29 30 31]]
[[ 4  5  6  7]
 [ 8  9 10 11]]
[[ 0  1  2  3]
 [12 13 14 15]
 [20 21 22 23]]
[ 0 15 22]
[[ 0  2  3]
 [12 14 15]
 [20 22 23]]

#转置
a = np.arange(24).reshape((2,3,4))
print(a)
b = a.T
print(b,b.shape)

[[[ 0  1  2  3]
  [ 4  5  6  7]
  [ 8  9 10 11]]

 [[12 13 14 15]
  [16 17 18 19]
  [20 21 22 23]]]
[[[ 0 12]
  [ 4 16]
  [ 8 20]]

 [[ 1 13]
  [ 5 17]
  [ 9 21]]

 [[ 2 14]
  [ 6 18]
  [10 22]]

 [[ 3 15]
  [ 7 19]
  [11 23]]] (4, 3, 2)

#函数
arr = np.array([-2,2,4,-3,100])
a = np.fabs(arr)
print(np.fabs(arr))#绝对值
print(np.sqrt(a))
print(np.square(arr))
print(np.sign(arr))
print(np.modf(arr))
print(np.nan)
print(np.inf)

[   2.    2.    4.    3.  100.]
[  1.41421356   1.41421356   2.           1.73205081  10.        ]
[    4     4    16     9 10000]
[-1  1  1 -1  1]
(array([-0.,  0.,  0., -0.,  0.]), array([  -2.,    2.,    4.,   -3.,  100.]))
nan
inf

#二元函数  dot  mod(数组相同)  greater (比较运算) logical_and()逻辑与或等
print(np.power(arr,2))
a = np.random.randint(2,100,(3,4,5))
print(a)
print(a.min(),a.max(),a.mean(),a.std(),a.sum())
#对一行与一列的聚合
print(a.max(axis=0))#0列的方向
print(a.max(axis=1))#1行的方向

[    4     4    16     9 10000]
[[[61 50 33 85 52]
  [88 27 33 49 87]
  [61 19 96 35 47]
  [45 93 60 73 14]]

 [[45 61 93 63 60]
  [ 5 97 87 82  8]
  [82 94 15 71 54]
  [15 34 71 33 23]]

 [[90 96 10 55 69]
  [71 46 48 80 41]
  [25 93 85 34 25]
  [91 34 85 98 82]]]
5 98 57.65 27.5201895585 3459

# 三元函数 np.where x if condition else y
a = np.array([[3,5],[2,8]])
b = np.array([[1,6],[4,3]])
print(a)
print(b)
c = a > b
print(a[c])
print(np.where(c,a,b))

[[3 5]
 [2 8]]
[[1 6]
 [4 3]]
[3 8]
[[3 6]
 [4 8]]

#Series
import numpy as np
import pandas as pd
# Series DataFrame
#将np中的数组转化为Series,就是一维数组上加上了索引，同时这个索引也可以自定义
arr = np.array([1,3,6,np.nan,8,9])
s1 = pd.Series(arr)
print(s1)
s1.index = [u'中国','vf','vfd','cds','cc']

0    1.0
1    3.0
2    6.0
3    NaN
4    8.0
5    9.0
dtype: float64

# 用列表创建Series
a = [5,6,9,8,7]
s2 = pd.Series(a)
print(s2)
# 用字典的方式创建索引
dict1 = {'语文':88,'数学':96}
s3 = pd.Series(dict1)
print(s3)

0    5
1    6
2    9
3    8
4    7
dtype: int64
数学    96
语文    88
dtype: int64

# 通过索引和下标创建Series对象
#index  index 放到Series中可以多加，同时Series之间的相加会根据索引进行自动对齐
print(s1[3:])
print(s2[2:5])

3    NaN
4    8.0
5    9.0
dtype: float64
2    9
3    8
4    7
dtype: int64

#DataFrame 创建
import pandas as pd
import numpy  as np
# 通过二维数组和字典进行创建,加入了行和列的索引,由Series组成的字典
arr = [['dd',22],['as',12],['cds',88]]
df = pd.DataFrame(arr)
print(df)
print(df.columns)
#通过字典方式创建,字典中的values必须是一维数组的形式,必须是同样大小的一维数组，也可以是单个数字,单个数字表示所有相同
dict1 = {'语文':[88,90,16],'数学':[90,86,36],'英语':[85,26,99],'政治':88}
df1 = pd.DataFrame(dict1)
print(df1)

     0   1
0   dd  22
1   as  12
2  cds  88
RangeIndex(start=0, stop=2, step=1)
   政治  数学  英语  语文
0  88  90  85  88
1  88  86  26  90
2  88  36  99  16

#对DataFrame的操作是先取列再取行与ndarray不同
#print(df1['0'])取行是错的
print(df1['语文'])
print(df1['语文'][0])
#当然取行也行，但是需要使用ix属性
print(df1.ix[0])#注意ix后不加括号
print(df1.ix[0]['语文'])

0    88
1    90
2    16
Name: 语文, dtype: int64
88
政治    88
数学    90
英语    85
语文    88
Name: 0, dtype: int64
88

# 切片操作,同时他可以取到边值
print(df1.ix[1:2,1])
df1['语文'][0] = np.nan
print(df1)#数据类型也改变
df1.ix[0]['语文'] = np.nan#无法改变
print(df1)

1    86
2    36
Name: 数学, dtype: int64
   政治  数学  英语  语文
0  88  90  85  88
1  88  86  26  90
2  88  36  99  16
   政治  数学  英语  语文
0  88  90  85  88
1  88  86  26  90
2  88  36  99  16

#pd.read_csv pd.read_json   pd.isnull pd.notnull
#pd.dropna()  fillna()  pd.describe()所有的数据描述unique   values_count  .isin([11])
#多层索引，可以给索引设置二维数组
# 取值的新方法
print(df1['profile'][15])
print(df1.ix[15]['profile'])
#iloc对下标值的操作
#loc对索引值的操作

import  matplotlib.pyplot as plt
import  numpy   as   np

# 生成数据
dataOut = np.arange(24).reshape((4,6))
# 保存数据
np.savetxt('data.txt',dataOut,fmt='%.2e')
# 读取数据
data = np.loadtxt('data.txt')
print(data)
y = np.random.randint(1,11,5)
print(y)
x = np.arange(len(y))
print(x)
plt.plot(x,y,color='r')
plt.bar(x,y,color='g')
#plt.pie(y)
plt.show()

[[  0.   1.   2.   3.   4.   5.]
 [  6.   7.   8.   9.  10.  11.]
 [ 12.  13.  14.  15.  16.  17.]
 [ 18.  19.  20.  21.  22.  23.]]
[ 1 10  9  6  7]
[0 1 2 3 4]

#中文显示问题解决
plt.rcParams['font.sans-serif']='SimHei'
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
mpl.rcParams['font.family'] = 'sans-serif'
mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['axes.unicode_minus'] = False

x = np.arange(-5,5)
y = np.sin(x)
print(x,y)
#颜色 线条  标记
plt.plot(x,y,color='r',linestyle='--',marker='o')
plt.show()

[-5 -4 -3 -2 -1  0  1  2  3  4] [ 0.95892427  0.7568025  -0.14112001 -0.90929743 -0.84147098  0.
  0.84147098  0.90929743  0.14112001 -0.7568025 ]

# 线程
import time
import threading
def music(name,loop):
    for i in range(loop):
        print('在听音乐：%s 时间 %s'%(name,time.ctime()))
        time.sleep(1)
def movie(name,loop):
    for i in range(loop):
        print('在看电影：%s 时间 %s'%(name,time.ctime()))
        time.sleep(1)
# 1.创建线程
t1 = threading.Thread(target=music,args=('模特',3))
t2 = threading.Thread(target=movie,args=('速度',5))
# 线程守护,设置线程守护需要在线程启动之前
t1.setDaemon(True)
t2.setDaemon(True)
t1.start()
t2.start()
# t1.join()
# t2.join()
# 主线程优先级高
# print('任务结束了 %s'%(time.ctime()))

print('任务结束了 %s'%(time.ctime()))

在听音乐：模特 时间 Thu Aug 30 16:25:07 2018
任务结束了 Thu Aug 30 16:25:07 2018
在看电影：速度 时间 Thu Aug 30 16:25:07 2018
在听音乐：模特 时间 Thu Aug 30 16:25:08 2018
在看电影：速度 时间 Thu Aug 30 16:25:08 2018
在听音乐：模特 时间 Thu Aug 30 16:25:09 2018
在看电影：速度 时间 Thu Aug 30 16:25:09 2018
在看电影：速度 时间 Thu Aug 30 16:25:10 2018
在看电影：速度 时间 Thu Aug 30 16:25:11 2018

balance = 0
lock = threading.Lock()
def change(n):
    global balance
    balance += n
    balance -= n

def run_thread(n):
    for i in range(1000000):
        lock.acquire()
        try:
            change(n)
        finally:
            lock.release()
        
#线程
t1 = threading.Thread(target=run_thread,args=(4,))
t2 = threading.Thread(target=run_thread,args=(5,))

t1.start()
t2.start()

t1.join()
t2.join()

print(balance)

0

import time
import multprocessing

#单进程
def work_1(filename,n):
    print('work_1 start')
    for i in range(n):
        with open(filename,'a') as f_obj:
            f_obj.write('www.ai111.com.cn \n')
            time.sleep(1)
    print('work_1 end')

def work_2(filename,n):
    print('work_2 start')
    for i in range(n):
        with open(filename,'a') as f_obj:
            f_obj.write('www.ai111.com.cn \n')
            time.sleep(1)
    print('work_2 end')

work_1('zhang.txt',3)
work_2('xiao.txt',4)
``



```python
# 同步执行  异步执行
p1 = multprocessing.Process(target=work_1,args=('zhang.txt',3))
p2 = multprocessing.Process(target=work_1,args=('zhang.txt',4))
p1.start()
p2.start()
#异步
lock = multiprocessing.Lock()
lock.acquire()
lock.release()

### 进程池
import os
import multiprocessing
import time
def work(n):
    print('run work(%s) start,word id :%s'%(n,os.getpid))
    time.sleep(2)
    print('run work(%s) end,word id :%s'%(n,os.getpid))
print('父进程 id:%s'%os.getpid())
#创建一个进程池，包含两个进程
p = multiprocessing.Pool(2)
#提交5个任务去进程池
for i in range(5):
    #同步提交
    p.apply_async(work,args=(i,))
    #异步提交
    p.apply(work,args=(i,))

    
p.close()
p.join()

# 通过队列和管道 Queue and Pipe
import time
import multiprocessingcessing

def put(q):
    for value in ['a','b','c']:
        print("发送 %s 到 queue"%value)
        q.put(value)
        time.sleep(1)
def get(q):
    while True:
        value = q.get(True)
        print('从queue中取到数据：%s'%value)

q = multiprocessing.Queue()

pWrite = multiprocessing.Process(target=put,args=(q,))
pReader = multiprocessing.Process(target=get,args=(q,))

pWrite.start()
pReader.start()

# 通过队列和管道 Queue and Pipe
import time
import multiprocessingcessing
#管道的方式
def put(q):
    for value in ['a','b','c']:
        print("发送 %s 到 queue"%value)
        #q.put(value)
        p[1].send(value)
        time.sleep(1)
def get(q):
    while True:
        #value = q.get(True)
        value = p[0].recv()
        print('从queue中取到数据：%s'%value)
# true全双工  false半双工
p = multiprocessing.Pipe(duplex=False)

pWrite = multiprocessing.Process(target=put,args=(q,))
pReader = multiprocessing.Process(target=get,args=(q,))

pWrite.start()
pReader.start()

import time
import threading
import queue

q = queue.Queue(maxsize=10)
def producer(name):
    count=1
    while True: