Pandas 2.2 中文官方教程和指南（十·二）

In [507]: index = pd.MultiIndex(
 .....:   levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
 .....:   codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
 .....:   names=["foo", "bar"],
 .....: )
 .....: 

In [508]: df_mi = pd.DataFrame(np.random.randn(10, 3), index=index, columns=["A", "B", "C"])

In [509]: df_mi
Out[509]: 
 A         B         C
foo bar 
foo one   -1.303456 -0.642994 -0.649456
 two    1.012694  0.414147  1.950460
 three  1.094544 -0.802899 -0.583343
bar one    0.410395  0.618321  0.560398
 two    1.434027 -0.033270  0.343197
baz two   -1.646063 -0.695847 -0.429156
 three -0.244688 -1.428229 -0.138691
qux one    1.866184 -1.446617  0.036660
 two   -1.660522  0.929553 -1.298649
 three  3.565769  0.682402  1.041927

In [510]: store.append("df_mi", df_mi)

In [511]: store.select("df_mi")
Out[511]: 
 A         B         C
foo bar 
foo one   -1.303456 -0.642994 -0.649456
 two    1.012694  0.414147  1.950460
 three  1.094544 -0.802899 -0.583343
bar one    0.410395  0.618321  0.560398
 two    1.434027 -0.033270  0.343197
baz two   -1.646063 -0.695847 -0.429156
 three -0.244688 -1.428229 -0.138691
qux one    1.866184 -1.446617  0.036660
 two   -1.660522  0.929553 -1.298649
 three  3.565769  0.682402  1.041927

# the levels are automatically included as data columns
In [512]: store.select("df_mi", "foo=bar")
Out[512]: 
 A         B         C
foo bar 
bar one  0.410395  0.618321  0.560398
 two  1.434027 -0.033270  0.343197 

string = "HolyMoly'"
store.select("df", "index == string") 

string = "HolyMoly'"
store.select('df', f'index == {string}') 

store.select("df", "index == %r" % string) 

In [513]: dfq = pd.DataFrame(
 .....:    np.random.randn(10, 4),
 .....:    columns=list("ABCD"),
 .....:    index=pd.date_range("20130101", periods=10),
 .....: )
 .....: 

In [514]: store.append("dfq", dfq, format="table", data_columns=True) 

In [515]: store.select("dfq", "index>pd.Timestamp('20130104') & columns=['A', 'B']")
Out[515]: 
 A         B
2013-01-05 -0.830545 -0.457071
2013-01-06  0.431186  1.049421
2013-01-07  0.617509 -0.811230
2013-01-08  0.947422 -0.671233
2013-01-09 -0.183798 -1.211230
2013-01-10  0.361428  0.887304 

In [516]: store.select("dfq", where="A>0 or C>0")
Out[516]: 
 A         B         C         D
2013-01-02  0.658179  0.362814 -0.917897  0.010165
2013-01-03  0.905122  1.848731 -1.184241  0.932053
2013-01-05 -0.830545 -0.457071  1.565581  1.148032
2013-01-06  0.431186  1.049421  0.383309  0.595013
2013-01-07  0.617509 -0.811230 -2.088563 -1.393500
2013-01-08  0.947422 -0.671233 -0.847097 -1.187785
2013-01-10  0.361428  0.887304  0.266457 -0.399641 

In [517]: store.select("df", "columns=['A', 'B']")
Out[517]: 
 A         B
2000-01-01  0.858644 -0.851236
2000-01-02 -0.080372 -1.268121
2000-01-03  0.816983  1.965656
2000-01-04  0.712795 -0.062433
2000-01-05 -0.298721 -1.988045
2000-01-06  1.103675  1.382242
2000-01-07 -0.729161 -0.142928
2000-01-08 -1.005977  0.465222 

In [518]: from datetime import timedelta

In [519]: dftd = pd.DataFrame(
 .....:    {
 .....:        "A": pd.Timestamp("20130101"),
 .....:        "B": [
 .....:            pd.Timestamp("20130101") + timedelta(days=i, seconds=10)
 .....:            for i in range(10)
 .....:        ],
 .....:    }
 .....: )
 .....: 

In [520]: dftd["C"] = dftd["A"] - dftd["B"]

In [521]: dftd
Out[521]: 
 A                   B                  C
0 2013-01-01 2013-01-01 00:00:10  -1 days +23:59:50
1 2013-01-01 2013-01-02 00:00:10  -2 days +23:59:50
2 2013-01-01 2013-01-03 00:00:10  -3 days +23:59:50
3 2013-01-01 2013-01-04 00:00:10  -4 days +23:59:50
4 2013-01-01 2013-01-05 00:00:10  -5 days +23:59:50
5 2013-01-01 2013-01-06 00:00:10  -6 days +23:59:50
6 2013-01-01 2013-01-07 00:00:10  -7 days +23:59:50
7 2013-01-01 2013-01-08 00:00:10  -8 days +23:59:50
8 2013-01-01 2013-01-09 00:00:10  -9 days +23:59:50
9 2013-01-01 2013-01-10 00:00:10 -10 days +23:59:50

In [522]: store.append("dftd", dftd, data_columns=True)

In [523]: store.select("dftd", "C<'-3.5D'")
Out[523]: 
 A                   B                  C
4 1970-01-01 00:00:01.356998400 2013-01-05 00:00:10  -5 days +23:59:50
5 1970-01-01 00:00:01.356998400 2013-01-06 00:00:10  -6 days +23:59:50
6 1970-01-01 00:00:01.356998400 2013-01-07 00:00:10  -7 days +23:59:50
7 1970-01-01 00:00:01.356998400 2013-01-08 00:00:10  -8 days +23:59:50
8 1970-01-01 00:00:01.356998400 2013-01-09 00:00:10  -9 days +23:59:50
9 1970-01-01 00:00:01.356998400 2013-01-10 00:00:10 -10 days +23:59:50 
```#### 查询 MultiIndex

通过使用级别的名称可以实现从 `MultiIndex` 中选择。

```py
In [524]: df_mi.index.names
Out[524]: FrozenList(['foo', 'bar'])

In [525]: store.select("df_mi", "foo=baz and bar=two")
Out[525]: 
 A         B         C
foo bar 
baz two -1.646063 -0.695847 -0.429156 

In [526]: index = pd.MultiIndex(
 .....:    levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
 .....:    codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
 .....: )
 .....: 

In [527]: df_mi_2 = pd.DataFrame(np.random.randn(10, 3), index=index, columns=["A", "B", "C"])

In [528]: df_mi_2
Out[528]: 
 A         B         C
foo one   -0.219582  1.186860 -1.437189
 two    0.053768  1.872644 -1.469813
 three -0.564201  0.876341  0.407749
bar one   -0.232583  0.179812  0.922152
 two   -1.820952 -0.641360  2.133239
baz two   -0.941248 -0.136307 -1.271305
 three -0.099774 -0.061438 -0.845172
qux one    0.465793  0.756995 -0.541690
 two   -0.802241  0.877657 -2.553831
 three  0.094899 -2.319519  0.293601

In [529]: store.append("df_mi_2", df_mi_2)

# the levels are automatically included as data columns with keyword level_n
In [530]: store.select("df_mi_2", "level_0=foo and level_1=two")
Out[530]: 
 A         B         C
foo two  0.053768  1.872644 -1.469813 

# we have automagically already created an index (in the first section)
In [531]: i = store.root.df.table.cols.index.index

In [532]: i.optlevel, i.kind
Out[532]: (6, 'medium')

# change an index by passing new parameters
In [533]: store.create_table_index("df", optlevel=9, kind="full")

In [534]: i = store.root.df.table.cols.index.index

In [535]: i.optlevel, i.kind
Out[535]: (9, 'full') 

In [536]: df_1 = pd.DataFrame(np.random.randn(10, 2), columns=list("AB"))

In [537]: df_2 = pd.DataFrame(np.random.randn(10, 2), columns=list("AB"))

In [538]: st = pd.HDFStore("appends.h5", mode="w")

In [539]: st.append("df", df_1, data_columns=["B"], index=False)

In [540]: st.append("df", df_2, data_columns=["B"], index=False)

In [541]: st.get_storer("df").table
Out[541]: 
/df/table (Table(20,)) ''
 description := {
 "index": Int64Col(shape=(), dflt=0, pos=0),
 "values_block_0": Float64Col(shape=(1,), dflt=0.0, pos=1),
 "B": Float64Col(shape=(), dflt=0.0, pos=2)}
 byteorder := 'little'
 chunkshape := (2730,) 

In [542]: st.create_table_index("df", columns=["B"], optlevel=9, kind="full")

In [543]: st.get_storer("df").table
Out[543]: 
/df/table (Table(20,)) ''
 description := {
 "index": Int64Col(shape=(), dflt=0, pos=0),
 "values_block_0": Float64Col(shape=(1,), dflt=0.0, pos=1),
 "B": Float64Col(shape=(), dflt=0.0, pos=2)}
 byteorder := 'little'
 chunkshape := (2730,)
 autoindex := True
 colindexes := {
 "B": Index(9, fullshuffle, zlib(1)).is_csi=True}

In [544]: st.close() 

In [545]: df_dc = df.copy()

In [546]: df_dc["string"] = "foo"

In [547]: df_dc.loc[df_dc.index[4:6], "string"] = np.nan

In [548]: df_dc.loc[df_dc.index[7:9], "string"] = "bar"

In [549]: df_dc["string2"] = "cool"

In [550]: df_dc.loc[df_dc.index[1:3], ["B", "C"]] = 1.0

In [551]: df_dc
Out[551]: 
 A         B         C string string2
2000-01-01  0.858644 -0.851236  1.058006    foo    cool
2000-01-02 -0.080372  1.000000  1.000000    foo    cool
2000-01-03  0.816983  1.000000  1.000000    foo    cool
2000-01-04  0.712795 -0.062433  0.736755    foo    cool
2000-01-05 -0.298721 -1.988045  1.475308    NaN    cool
2000-01-06  1.103675  1.382242 -0.650762    NaN    cool
2000-01-07 -0.729161 -0.142928 -1.063038    foo    cool
2000-01-08 -1.005977  0.465222 -0.094517    bar    cool

# on-disk operations
In [552]: store.append("df_dc", df_dc, data_columns=["B", "C", "string", "string2"])

In [553]: store.select("df_dc", where="B > 0")
Out[553]: 
 A         B         C string string2
2000-01-02 -0.080372  1.000000  1.000000    foo    cool
2000-01-03  0.816983  1.000000  1.000000    foo    cool
2000-01-06  1.103675  1.382242 -0.650762    NaN    cool
2000-01-08 -1.005977  0.465222 -0.094517    bar    cool

# getting creative
In [554]: store.select("df_dc", "B > 0 & C > 0 & string == foo")
Out[554]: 
 A    B    C string string2
2000-01-02 -0.080372  1.0  1.0    foo    cool
2000-01-03  0.816983  1.0  1.0    foo    cool

# this is in-memory version of this type of selection
In [555]: df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")]
Out[555]: 
 A    B    C string string2
2000-01-02 -0.080372  1.0  1.0    foo    cool
2000-01-03  0.816983  1.0  1.0    foo    cool

# we have automagically created this index and the B/C/string/string2
# columns are stored separately as ``PyTables`` columns
In [556]: store.root.df_dc.table
Out[556]: 
/df_dc/table (Table(8,)) ''
 description := {
 "index": Int64Col(shape=(), dflt=0, pos=0),
 "values_block_0": Float64Col(shape=(1,), dflt=0.0, pos=1),
 "B": Float64Col(shape=(), dflt=0.0, pos=2),
 "C": Float64Col(shape=(), dflt=0.0, pos=3),
 "string": StringCol(itemsize=3, shape=(), dflt=b'', pos=4),
 "string2": StringCol(itemsize=4, shape=(), dflt=b'', pos=5)}
 byteorder := 'little'
 chunkshape := (1680,)
 autoindex := True
 colindexes := {
 "index": Index(6, mediumshuffle, zlib(1)).is_csi=False,
 "B": Index(6, mediumshuffle, zlib(1)).is_csi=False,
 "C": Index(6, mediumshuffle, zlib(1)).is_csi=False,
 "string": Index(6, mediumshuffle, zlib(1)).is_csi=False,
 "string2": Index(6, mediumshuffle, zlib(1)).is_csi=False} 

In [557]: for df in store.select("df", chunksize=3):
 .....:    print(df)
 .....: 
 A         B         C
2000-01-01  0.858644 -0.851236  1.058006
2000-01-02 -0.080372 -1.268121  1.561967
2000-01-03  0.816983  1.965656 -1.169408
 A         B         C
2000-01-04  0.712795 -0.062433  0.736755
2000-01-05 -0.298721 -1.988045  1.475308
2000-01-06  1.103675  1.382242 -0.650762
 A         B         C
2000-01-07 -0.729161 -0.142928 -1.063038
2000-01-08 -1.005977  0.465222 -0.094517 

for df in pd.read_hdf("store.h5", "df", chunksize=3):
    print(df) 

In [558]: dfeq = pd.DataFrame({"number": np.arange(1, 11)})

In [559]: dfeq
Out[559]: 
 number
0       1
1       2
2       3
3       4
4       5
5       6
6       7
7       8
8       9
9      10

In [560]: store.append("dfeq", dfeq, data_columns=["number"])

In [561]: def chunks(l, n):
 .....:    return [l[i: i + n] for i in range(0, len(l), n)]
 .....: 

In [562]: evens = [2, 4, 6, 8, 10]

In [563]: coordinates = store.select_as_coordinates("dfeq", "number=evens")

In [564]: for c in chunks(coordinates, 2):
 .....:    print(store.select("dfeq", where=c))
 .....: 
 number
1       2
3       4
 number
5       6
7       8
 number
9      10 

In [565]: store.select_column("df_dc", "index")
Out[565]: 
0   2000-01-01
1   2000-01-02
2   2000-01-03
3   2000-01-04
4   2000-01-05
5   2000-01-06
6   2000-01-07
7   2000-01-08
Name: index, dtype: datetime64[ns]

In [566]: store.select_column("df_dc", "string")
Out[566]: 
0    foo
1    foo
2    foo
3    foo
4    NaN
5    NaN
6    foo
7    bar
Name: string, dtype: object 

In [567]: df_coord = pd.DataFrame(
 .....:    np.random.randn(1000, 2), index=pd.date_range("20000101", periods=1000)
 .....: )
 .....: 

In [568]: store.append("df_coord", df_coord)

In [569]: c = store.select_as_coordinates("df_coord", "index > 20020101")

In [570]: c
Out[570]: 
Index([732, 733, 734, 735, 736, 737, 738, 739, 740, 741,
 ...
 990, 991, 992, 993, 994, 995, 996, 997, 998, 999],
 dtype='int64', length=268)

In [571]: store.select("df_coord", where=c)
Out[571]: 
 0         1
2002-01-02  0.007717  1.168386
2002-01-03  0.759328 -0.638934
2002-01-04 -1.154018 -0.324071
2002-01-05 -0.804551 -1.280593
2002-01-06 -0.047208  1.260503
...              ...       ...
2002-09-22 -1.139583  0.344316
2002-09-23 -0.760643 -1.306704
2002-09-24  0.059018  1.775482
2002-09-25  1.242255 -0.055457
2002-09-26  0.410317  2.194489

[268 rows x 2 columns] 
```##### 使用 where 掩码进行选择

有时候你的查询可能涉及创建一个要选择的行列表。通常这个`mask`会是一个索引操作的结果`index`。这个示例选择了一个 datetimeindex 中为 5 的月份。

```py
In [572]: df_mask = pd.DataFrame(
 .....:    np.random.randn(1000, 2), index=pd.date_range("20000101", periods=1000)
 .....: )
 .....: 

In [573]: store.append("df_mask", df_mask)

In [574]: c = store.select_column("df_mask", "index")

In [575]: where = c[pd.DatetimeIndex(c).month == 5].index

In [576]: store.select("df_mask", where=where)
Out[576]: 
 0         1
2000-05-01  1.479511  0.516433
2000-05-02 -0.334984 -1.493537
2000-05-03  0.900321  0.049695
2000-05-04  0.614266 -1.077151
2000-05-05  0.233881  0.493246
...              ...       ...
2002-05-27  0.294122  0.457407
2002-05-28 -1.102535  1.215650
2002-05-29 -0.432911  0.753606
2002-05-30 -1.105212  2.311877
2002-05-31  2.567296  2.610691

[93 rows x 2 columns] 

In [577]: store.get_storer("df_dc").nrows
Out[577]: 8 

In [578]: df_mt = pd.DataFrame(
 .....:    np.random.randn(8, 6),
 .....:    index=pd.date_range("1/1/2000", periods=8),
 .....:    columns=["A", "B", "C", "D", "E", "F"],
 .....: )
 .....: 

In [579]: df_mt["foo"] = "bar"

In [580]: df_mt.loc[df_mt.index[1], ("A", "B")] = np.nan

# you can also create the tables individually
In [581]: store.append_to_multiple(
 .....:    {"df1_mt": ["A", "B"], "df2_mt": None}, df_mt, selector="df1_mt"
 .....: )
 .....: 

In [582]: store
Out[582]: 
<class 'pandas.io.pytables.HDFStore'>
File path: store.h5

# individual tables were created
In [583]: store.select("df1_mt")
Out[583]: 
 A         B
2000-01-01  0.162291 -0.430489
2000-01-02       NaN       NaN
2000-01-03  0.429207 -1.099274
2000-01-04  1.869081 -1.466039
2000-01-05  0.092130 -1.726280
2000-01-06  0.266901 -0.036854
2000-01-07 -0.517871 -0.990317
2000-01-08 -0.231342  0.557402

In [584]: store.select("df2_mt")
Out[584]: 
 C         D         E         F  foo
2000-01-01 -2.502042  0.668149  0.460708  1.834518  bar
2000-01-02  0.130441 -0.608465  0.439872  0.506364  bar
2000-01-03 -1.069546  1.236277  0.116634 -1.772519  bar
2000-01-04  0.137462  0.313939  0.748471 -0.943009  bar
2000-01-05  0.836517  2.049798  0.562167  0.189952  bar
2000-01-06  1.112750 -0.151596  1.503311  0.939470  bar
2000-01-07 -0.294348  0.335844 -0.794159  1.495614  bar
2000-01-08  0.860312 -0.538674 -0.541986 -1.759606  bar

# as a multiple
In [585]: store.select_as_multiple(
 .....:    ["df1_mt", "df2_mt"],
 .....:    where=["A>0", "B>0"],
 .....:    selector="df1_mt",
 .....: )
 .....: 
Out[585]: 
Empty DataFrame
Columns: [A, B, C, D, E, F, foo]
Index: [] 

store_compressed = pd.HDFStore(
    "store_compressed.h5", complevel=9, complib="blosc:blosclz"
) 

store.append("df", df, complib="zlib", complevel=5) 

ptrepack --chunkshape=auto --propindexes --complevel=9 --complib=blosc in.h5 out.h5 

In [586]: dfcat = pd.DataFrame(
 .....:    {"A": pd.Series(list("aabbcdba")).astype("category"), "B": np.random.randn(8)}
 .....: )
 .....: 

In [587]: dfcat
Out[587]: 
 A         B
0  a -1.520478
1  a -1.069391
2  b -0.551981
3  b  0.452407
4  c  0.409257
5  d  0.301911
6  b -0.640843
7  a -2.253022

In [588]: dfcat.dtypes
Out[588]: 
A    category
B     float64
dtype: object

In [589]: cstore = pd.HDFStore("cats.h5", mode="w")

In [590]: cstore.append("dfcat", dfcat, format="table", data_columns=["A"])

In [591]: result = cstore.select("dfcat", where="A in ['b', 'c']")

In [592]: result
Out[592]: 
 A         B
2  b -0.551981
3  b  0.452407
4  c  0.409257
6  b -0.640843

In [593]: result.dtypes
Out[593]: 
A    category
B     float64
dtype: object 

In [594]: dfs = pd.DataFrame({"A": "foo", "B": "bar"}, index=list(range(5)))

In [595]: dfs
Out[595]: 
 A    B
0  foo  bar
1  foo  bar
2  foo  bar
3  foo  bar
4  foo  bar

# A and B have a size of 30
In [596]: store.append("dfs", dfs, min_itemsize=30)

In [597]: store.get_storer("dfs").table
Out[597]: 
/dfs/table (Table(5,)) ''
 description := {
 "index": Int64Col(shape=(), dflt=0, pos=0),
 "values_block_0": StringCol(itemsize=30, shape=(2,), dflt=b'', pos=1)}
 byteorder := 'little'
 chunkshape := (963,)
 autoindex := True
 colindexes := {
 "index": Index(6, mediumshuffle, zlib(1)).is_csi=False}

# A is created as a data_column with a size of 30
# B is size is calculated
In [598]: store.append("dfs2", dfs, min_itemsize={"A": 30})

In [599]: store.get_storer("dfs2").table
Out[599]: 
/dfs2/table (Table(5,)) ''
 description := {
 "index": Int64Col(shape=(), dflt=0, pos=0),
 "values_block_0": StringCol(itemsize=3, shape=(1,), dflt=b'', pos=1),
 "A": StringCol(itemsize=30, shape=(), dflt=b'', pos=2)}
 byteorder := 'little'
 chunkshape := (1598,)
 autoindex := True
 colindexes := {
 "index": Index(6, mediumshuffle, zlib(1)).is_csi=False,
 "A": Index(6, mediumshuffle, zlib(1)).is_csi=False} 

In [600]: dfss = pd.DataFrame({"A": ["foo", "bar", "nan"]})

In [601]: dfss
Out[601]: 
 A
0  foo
1  bar
2  nan

In [602]: store.append("dfss", dfss)

In [603]: store.select("dfss")
Out[603]: 
 A
0  foo
1  bar
2  NaN

# here you need to specify a different nan rep
In [604]: store.append("dfss2", dfss, nan_rep="_nan_")

In [605]: store.select("dfss2")
Out[605]: 
 A
0  foo
1  bar
2  nan 

In [606]: df = pd.DataFrame(
 .....:    {
 .....:        "a": list("abc"),
 .....:        "b": list(range(1, 4)),
 .....:        "c": np.arange(3, 6).astype("u1"),
 .....:        "d": np.arange(4.0, 7.0, dtype="float64"),
 .....:        "e": [True, False, True],
 .....:        "f": pd.Categorical(list("abc")),
 .....:        "g": pd.date_range("20130101", periods=3),
 .....:        "h": pd.date_range("20130101", periods=3, tz="US/Eastern"),
 .....:        "i": pd.date_range("20130101", periods=3, freq="ns"),
 .....:    }
 .....: )
 .....: 

In [607]: df
Out[607]: 
 a  b  c  ...          g                         h                             i
0  a  1  3  ... 2013-01-01 2013-01-01 00:00:00-05:00 2013-01-01 00:00:00.000000000
1  b  2  4  ... 2013-01-02 2013-01-02 00:00:00-05:00 2013-01-01 00:00:00.000000001
2  c  3  5  ... 2013-01-03 2013-01-03 00:00:00-05:00 2013-01-01 00:00:00.000000002

[3 rows x 9 columns]

In [608]: df.dtypes
Out[608]: 
a                        object
b                         int64
c                         uint8
d                       float64
e                          bool
f                      category
g                datetime64[ns]
h    datetime64[ns, US/Eastern]
i                datetime64[ns]
dtype: object 

In [609]: df.to_feather("example.feather") 

In [610]: result = pd.read_feather("example.feather")

In [611]: result
Out[611]: 
 a  b  c  ...          g                         h                             i
0  a  1  3  ... 2013-01-01 2013-01-01 00:00:00-05:00 2013-01-01 00:00:00.000000000
1  b  2  4  ... 2013-01-02 2013-01-02 00:00:00-05:00 2013-01-01 00:00:00.000000001
2  c  3  5  ... 2013-01-03 2013-01-03 00:00:00-05:00 2013-01-01 00:00:00.000000002

[3 rows x 9 columns]

# we preserve dtypes
In [612]: result.dtypes
Out[612]: 
a                        object
b                         int64
c                         uint8
d                       float64
e                          bool
f                      category
g                datetime64[ns]
h    datetime64[ns, US/Eastern]
i                datetime64[ns]
dtype: object 
```## Parquet

[Apache Parquet](https://parquet.apache.org/) 为数据框提供了分区的二进制列序列化。它旨在使数据框的读写高效，并使数据在数据分析语言之间的共享变得容易。Parquet 可以使用各种压缩技术来尽可能地缩小文件大小，同时保持良好的读取性能。

Parquet 旨在忠实地序列化和反序列化 `DataFrame`，支持所有 pandas 的数据类型，包括带有时区的日期时间等扩展数据类型。

几个注意事项。

+   不支持重复的列名和非字符串的列名。

+   `pyarrow` 引擎始终将索引写入输出，但 `fastparquet` 仅写入非默认索引。这个额外的列可能会给那些不希望看到它的非 pandas 消费者带来问题。您可以使用 `index` 参数强制包含或省略索引，而不管底层引擎如何。

+   如果指定了索引级别名称，则必须是字符串。

+   在 `pyarrow` 引擎中，非字符串类型的分类数据类型可以序列化为 parquet，但会反序列化为其原始数据类型。

+   `pyarrow` 引擎保留了具有字符串类型的分类数据类型的 `ordered` 标志。`fastparquet` 不保留 `ordered` 标志。

+   不支持的类型包括 `Interval` 和实际的 Python 对象类型。在尝试序列化时，这些将引发一个有用的错误消息。`Period` 类型在 pyarrow >= 0.16.0 中受支持。

+   `pyarrow` 引擎保留扩展数据类型，如可空整数和字符串数据类型（需要 pyarrow >= 0.16.0，并要求扩展类型实现所需的协议，请参阅扩展类型文档）。

您可以指定一个`engine`来指导序列化。这可以是`pyarrow`、`fastparquet`或`auto`中的一个。如果未指定引擎，则会检查`pd.options.io.parquet.engine`选项；如果这也是`auto`，则尝试`pyarrow`，并回退到`fastparquet`。

参阅[pyarrow](https://arrow.apache.org/docs/python/)和[fastparquet](https://fastparquet.readthedocs.io/en/latest/)的文档。

注意

这些引擎非常相似，几乎可以读/写完全相同的 Parquet 格式文件。`pyarrow>=8.0.0`支持时间间隔数据，`fastparquet>=0.1.4`支持时区感知日期时间。这些库之间的区别在于具有不同的底层依赖关系（`fastparquet`使用`numba`，而`pyarrow`使用 C 库）。

```py
In [613]: df = pd.DataFrame(
 .....:    {
 .....:        "a": list("abc"),
 .....:        "b": list(range(1, 4)),
 .....:        "c": np.arange(3, 6).astype("u1"),
 .....:        "d": np.arange(4.0, 7.0, dtype="float64"),
 .....:        "e": [True, False, True],
 .....:        "f": pd.date_range("20130101", periods=3),
 .....:        "g": pd.date_range("20130101", periods=3, tz="US/Eastern"),
 .....:        "h": pd.Categorical(list("abc")),
 .....:        "i": pd.Categorical(list("abc"), ordered=True),
 .....:    }
 .....: )
 .....: 

In [614]: df
Out[614]: 
 a  b  c    d      e          f                         g  h  i
0  a  1  3  4.0   True 2013-01-01 2013-01-01 00:00:00-05:00  a  a
1  b  2  4  5.0  False 2013-01-02 2013-01-02 00:00:00-05:00  b  b
2  c  3  5  6.0   True 2013-01-03 2013-01-03 00:00:00-05:00  c  c

In [615]: df.dtypes
Out[615]: 
a                        object
b                         int64
c                         uint8
d                       float64
e                          bool
f                datetime64[ns]
g    datetime64[ns, US/Eastern]
h                      category
i                      category
dtype: object 

In [616]: df.to_parquet("example_pa.parquet", engine="pyarrow")

In [617]: df.to_parquet("example_fp.parquet", engine="fastparquet") 

In [618]: result = pd.read_parquet("example_fp.parquet", engine="fastparquet")

In [619]: result = pd.read_parquet("example_pa.parquet", engine="pyarrow")

In [620]: result.dtypes
Out[620]: 
a                        object
b                         int64
c                         uint8
d                       float64
e                          bool
f                datetime64[ns]
g    datetime64[ns, US/Eastern]
h                      category
i                      category
dtype: object 

In [621]: result = pd.read_parquet("example_pa.parquet", engine="pyarrow", dtype_backend="pyarrow")

In [622]: result.dtypes
Out[622]: 
a                                      string[pyarrow]
b                                       int64[pyarrow]
c                                       uint8[pyarrow]
d                                      double[pyarrow]
e                                        bool[pyarrow]
f                               timestamp[ns][pyarrow]
g                timestamp[ns, tz=US/Eastern][pyarrow]
h    dictionary<values=string, indices=int32, order...
i    dictionary<values=string, indices=int32, order...
dtype: object 

In [623]: result = pd.read_parquet(
 .....:    "example_fp.parquet",
 .....:    engine="fastparquet",
 .....:    columns=["a", "b"],
 .....: )
 .....: 

In [624]: result = pd.read_parquet(
 .....:    "example_pa.parquet",
 .....:    engine="pyarrow",
 .....:    columns=["a", "b"],
 .....: )
 .....: 

In [625]: result.dtypes
Out[625]: 
a    object
b     int64
dtype: object 

In [626]: df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})

In [627]: df.to_parquet("test.parquet", engine="pyarrow") 

In [628]: df.to_parquet("test.parquet", index=False) 

In [629]: df = pd.DataFrame({"a": [0, 0, 1, 1], "b": [0, 1, 0, 1]})

In [630]: df.to_parquet(path="test", engine="pyarrow", partition_cols=["a"], compression=None) 

test
├── a=0
│   ├── 0bac803e32dc42ae83fddfd029cbdebc.parquet
│   └──  ...
└── a=1
    ├── e6ab24a4f45147b49b54a662f0c412a3.parquet
    └── ... 
```## ORC

与 parquet 格式类似，[ORC 格式](https://orc.apache.org/)是用于数据框的二进制列序列化。它旨在使数据框的读取效率更高。pandas 为 ORC 格式提供了读取器和写入器，`read_orc()`和`to_orc()`。这需要[pyarrow](https://arrow.apache.org/docs/python/)库。

警告

+   建议使用 conda 安装 pyarrow，因为 pyarrow 存在一些问题。

+   `to_orc()`需要 pyarrow>=7.0.0。

+   `read_orc()`和`to_orc()`目前尚不支持 Windows，您可以在安装可选依赖项中找到有效的环境。

+   有关支持的数据类型，请参考[Arrow 中支持的 ORC 功能](https://arrow.apache.org/docs/cpp/orc.html#data-types)。

+   目前，将数据框转换为 ORC 文件时，日期时间列中的时区信息不会被保留。

```py
In [631]: df = pd.DataFrame(
 .....:    {
 .....:        "a": list("abc"),
 .....:        "b": list(range(1, 4)),
 .....:        "c": np.arange(4.0, 7.0, dtype="float64"),
 .....:        "d": [True, False, True],
 .....:        "e": pd.date_range("20130101", periods=3),
 .....:    }
 .....: )
 .....: 

In [632]: df
Out[632]: 
 a  b    c      d          e
0  a  1  4.0   True 2013-01-01
1  b  2  5.0  False 2013-01-02
2  c  3  6.0   True 2013-01-03

In [633]: df.dtypes
Out[633]: 
a            object
b             int64
c           float64
d              bool
e    datetime64[ns]
dtype: object 

In [634]: df.to_orc("example_pa.orc", engine="pyarrow") 

In [635]: result = pd.read_orc("example_pa.orc")

In [636]: result.dtypes
Out[636]: 
a            object
b             int64
c           float64
d              bool
e    datetime64[ns]
dtype: object 

In [637]: result = pd.read_orc(
 .....:    "example_pa.orc",
 .....:    columns=["a", "b"],
 .....: )
 .....: 

In [638]: result.dtypes
Out[638]: 
a    object
b     int64
dtype: object 
```## SQL 查询

`pandas.io.sql`模块提供了一组查询包装器，旨在促进数据检索并减少对特定于数据库的 API 的依赖。

如有可能，用户可能首选选择[Apache Arrow ADBC](https://arrow.apache.org/adbc/current/index.html)驱动程序。这些驱动程序应提供最佳性能、空值处理和类型检测。

> 版本 2.2.0 中的新功能：增加了对 ADBC 驱动程序的本机支持

有关 ADBC 驱动程序及其开发状态的完整列表，请参阅[ADBC 驱动程序实现状态](https://arrow.apache.org/adbc/current/driver/status.html)文档。

如果没有 ADBC 驱动程序或缺少功能，则用户应选择安装 SQLAlchemy 以及其数据库驱动程序库。这些驱动程序的示例是[psycopg2](https://www.psycopg.org/)用于 PostgreSQL 或[pymysql](https://github.com/PyMySQL/PyMySQL)用于 MySQL。对于[SQLite](https://docs.python.org/3/library/sqlite3.html)，这在 Python 的标准库中默认包含。您可以在[SQLAlchemy 文档](https://docs.sqlalchemy.org/en/latest/dialects/index.html)中找到每个 SQL 方言支持的驱动程序的概述。

如果未安装 SQLAlchemy，可以使用[`sqlite3.Connection`](https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection "(在 Python v3.12 中)")代替 SQLAlchemy 引擎、连接或 URI 字符串。

还可以查看一些烹饪书示例以获取一些高级策略。

关键函数包括：

| `read_sql_table`(table_name, con[, schema, ...]) | 将 SQL 数据库表读取到数据框中。 |
| --- | --- |
| `read_sql_query`(sql, con[, index_col, ...]) | 将 SQL 查询读取到数据框中。 |
| `read_sql`(sql, con[, index_col, ...]) | 将 SQL 查询或数据库表读取到数据框中。 |
| `DataFrame.to_sql`(name, con, *[, schema, ...]) | 将存储在数据框中的记录写入 SQL 数据库。 |

注意

函数`read_sql()`是对`read_sql_table()`和`read_sql_query()`（以及向后兼容性）的便捷包装，并根据提供的输入（数据库表名或 SQL 查询）委托给特定函数。如果表名包含特殊字符，则不需要对表名加引号。

在以下示例中，我们使用 [SQlite](https://www.sqlite.org/index.html) SQL 数据库引擎。您可以使用一个临时 SQLite 数据库，其中数据存储在“内存”中。

要使用 ADBC 驱动程序进行连接，您需要使用包管理器安装`adbc_driver_sqlite`。安装后，您可以使用 ADBC 驱动程序提供的 DBAPI 接口连接到数据库。

```py
import adbc_driver_sqlite.dbapi as sqlite_dbapi

# Create the connection
with sqlite_dbapi.connect("sqlite:///:memory:") as conn:
     df = pd.read_sql_table("data", conn) 

In [639]: from sqlalchemy import create_engine

# Create your engine.
In [640]: engine = create_engine("sqlite:///:memory:") 

with engine.connect() as conn, conn.begin():
    data = pd.read_sql_table("data", conn) 

In [641]: import datetime

In [642]: c = ["id", "Date", "Col_1", "Col_2", "Col_3"]

In [643]: d = [
 .....:    (26, datetime.datetime(2010, 10, 18), "X", 27.5, True),
 .....:    (42, datetime.datetime(2010, 10, 19), "Y", -12.5, False),
 .....:    (63, datetime.datetime(2010, 10, 20), "Z", 5.73, True),
 .....: ]
 .....: 

In [644]: data = pd.DataFrame(d, columns=c)

In [645]: data
Out[645]: 
 id       Date Col_1  Col_2  Col_3
0  26 2010-10-18     X  27.50   True
1  42 2010-10-19     Y -12.50  False
2  63 2010-10-20     Z   5.73   True

In [646]: data.to_sql("data", con=engine)
Out[646]: 3 

In [647]: data.to_sql("data_chunked", con=engine, chunksize=1000)
Out[647]: 3 

# for roundtripping
with pg_dbapi.connect(uri) as conn:
    df2 = pd.read_sql("pandas_table", conn, dtype_backend="pyarrow") 

In [648]: from sqlalchemy.types import String

In [649]: data.to_sql("data_dtype", con=engine, dtype={"Col_1": String})
Out[649]: 3 

# Alternative to_sql() *method* for DBs that support COPY FROM
import csv
from io import StringIO

def psql_insert_copy(table, conn, keys, data_iter):
  """
 Execute SQL statement inserting data

 Parameters
 ----------
 table : pandas.io.sql.SQLTable
 conn : sqlalchemy.engine.Engine or sqlalchemy.engine.Connection
 keys : list of str
 Column names
 data_iter : Iterable that iterates the values to be inserted
 """
    # gets a DBAPI connection that can provide a cursor
    dbapi_conn = conn.connection
    with dbapi_conn.cursor() as cur:
        s_buf = StringIO()
        writer = csv.writer(s_buf)
        writer.writerows(data_iter)
        s_buf.seek(0)

        columns = ', '.join(['"{}"'.format(k) for k in keys])
        if table.schema:
            table_name = '{}.{}'.format(table.schema, table.name)
        else:
            table_name = table.name

        sql = 'COPY {} ({}) FROM STDIN WITH CSV'.format(
            table_name, columns)
        cur.copy_expert(sql=sql, file=s_buf) 

In [650]: pd.read_sql_table("data", engine)
Out[650]: 
 index  id       Date Col_1  Col_2  Col_3
0      0  26 2010-10-18     X  27.50   True
1      1  42 2010-10-19     Y -12.50  False
2      2  63 2010-10-20     Z   5.73   True 

In [651]: pd.read_sql_table("data", engine, index_col="id")
Out[651]: 
 index       Date Col_1  Col_2  Col_3
id 
26      0 2010-10-18     X  27.50   True
42      1 2010-10-19     Y -12.50  False
63      2 2010-10-20     Z   5.73   True

In [652]: pd.read_sql_table("data", engine, columns=["Col_1", "Col_2"])
Out[652]: 
 Col_1  Col_2
0     X  27.50
1     Y -12.50
2     Z   5.73 

In [653]: pd.read_sql_table("data", engine, parse_dates=["Date"])
Out[653]: 
 index  id       Date Col_1  Col_2  Col_3
0      0  26 2010-10-18     X  27.50   True
1      1  42 2010-10-19     Y -12.50  False
2      2  63 2010-10-20     Z   5.73   True 

pd.read_sql_table("data", engine, parse_dates={"Date": "%Y-%m-%d"})
pd.read_sql_table(
    "data",
    engine,
    parse_dates={"Date": {"format": "%Y-%m-%d %H:%M:%S"}},
) 

df.to_sql(name="table", con=engine, schema="other_schema")
pd.read_sql_table("table", engine, schema="other_schema") 

In [654]: pd.read_sql_query("SELECT * FROM data", engine)
Out[654]: 
 index  id                        Date Col_1  Col_2  Col_3
0      0  26  2010-10-18 00:00:00.000000     X  27.50      1
1      1  42  2010-10-19 00:00:00.000000     Y -12.50      0
2      2  63  2010-10-20 00:00:00.000000     Z   5.73      1 

In [655]: pd.read_sql_query("SELECT id, Col_1, Col_2 FROM data WHERE id = 42;", engine)
Out[655]: 
 id Col_1  Col_2
0  42     Y  -12.5 

In [656]: df = pd.DataFrame(np.random.randn(20, 3), columns=list("abc"))

In [657]: df.to_sql(name="data_chunks", con=engine, index=False)
Out[657]: 20 

In [658]: for chunk in pd.read_sql_query("SELECT * FROM data_chunks", engine, chunksize=5):
 .....:    print(chunk)
 .....: 
 a         b         c
0 -0.395347 -0.822726 -0.363777
1  1.676124 -0.908102 -1.391346
2 -1.094269  0.278380  1.205899
3  1.503443  0.932171 -0.709459
4 -0.645944 -1.351389  0.132023
 a         b         c
0  0.210427  0.192202  0.661949
1  1.690629 -1.046044  0.618697
2 -0.013863  1.314289  1.951611
3 -1.485026  0.304662  1.194757
4 -0.446717  0.528496 -0.657575
 a         b         c
0 -0.876654  0.336252  0.172668
1  0.337684 -0.411202 -0.828394
2 -0.244413  1.094948  0.087183
3  1.125934 -1.480095  1.205944
4 -0.451849  0.452214 -2.208192
 a         b         c
0 -2.061019  0.044184 -0.017118
1  1.248959 -0.675595 -1.908296
2 -0.125934  1.491974  0.648726
3  0.391214  0.438609  1.634248
4  1.208707 -1.535740  1.620399 

from sqlalchemy import create_engine

engine = create_engine("postgresql://scott:tiger@localhost:5432/mydatabase")

engine = create_engine("mysql+mysqldb://scott:tiger@localhost/foo")

engine = create_engine("oracle://scott:[[email protected]](/cdn-cgi/l/email-protection):1521/sidname")

engine = create_engine("mssql+pyodbc://mydsn")

# sqlite://<nohostname>/<path>
# where <path> is relative:
engine = create_engine("sqlite:///foo.db")

# or absolute, starting with a slash:
engine = create_engine("sqlite:absolute/path/to/foo.db") 

In [659]: import sqlalchemy as sa

In [660]: pd.read_sql(
 .....:    sa.text("SELECT * FROM data where Col_1=:col1"), engine, params={"col1": "X"}
 .....: )
 .....: 
Out[660]: 
 index  id                        Date Col_1  Col_2  Col_3
0      0  26  2010-10-18 00:00:00.000000     X   27.5      1 

In [661]: metadata = sa.MetaData()

In [662]: data_table = sa.Table(
 .....:    "data",
 .....:    metadata,
 .....:    sa.Column("index", sa.Integer),
 .....:    sa.Column("Date", sa.DateTime),
 .....:    sa.Column("Col_1", sa.String),
 .....:    sa.Column("Col_2", sa.Float),
 .....:    sa.Column("Col_3", sa.Boolean),
 .....: )
 .....: 

In [663]: pd.read_sql(sa.select(data_table).where(data_table.c.Col_3 is True), engine)
Out[663]: 
Empty DataFrame
Columns: [index, Date, Col_1, Col_2, Col_3]
Index: [] 

In [664]: import datetime as dt

In [665]: expr = sa.select(data_table).where(data_table.c.Date > sa.bindparam("date"))

In [666]: pd.read_sql(expr, engine, params={"date": dt.datetime(2010, 10, 18)})
Out[666]: 
 index       Date Col_1  Col_2  Col_3
0      1 2010-10-19     Y -12.50  False
1      2 2010-10-20     Z   5.73   True 

import sqlite3

con = sqlite3.connect(":memory:") 

data.to_sql("data", con)
pd.read_sql_query("SELECT * FROM data", con) 
```## Google BigQuery

`pandas-gbq` 包提供了与 Google BigQuery 读写的功能。

pandas 与这个外部包集成。如果安装了 `pandas-gbq`，则可以使用 pandas 方法 `pd.read_gbq` 和 `DataFrame.to_gbq`，这将调用 `pandas-gbq` 中的相应函数。

完整文档可以在[这里](https://pandas-gbq.readthedocs.io/en/latest/)找到。  ## Stata 格式

### 写入到 Stata 格式

方法 `DataFrame.to_stata()` 将 DataFrame 写入 .dta 文件。此文件的格式版本始终为 115（Stata 12）。

```py
In [667]: df = pd.DataFrame(np.random.randn(10, 2), columns=list("AB"))

In [668]: df.to_stata("stata.dta") 

In [669]: pd.read_stata("stata.dta")
Out[669]: 
 index         A         B
0      0 -0.165614  0.490482
1      1 -0.637829  0.067091
2      2 -0.242577  1.348038
3      3  0.647699 -0.644937
4      4  0.625771  0.918376
5      5  0.401781 -1.488919
6      6 -0.981845 -0.046882
7      7 -0.306796  0.877025
8      8 -0.336606  0.624747
9      9 -1.582600  0.806340 

In [670]: with pd.read_stata("stata.dta", chunksize=3) as reader:
 .....:    for df in reader:
 .....:        print(df.shape)
 .....: 
(3, 3)
(3, 3)
(3, 3)
(1, 3) 

In [671]: with pd.read_stata("stata.dta", iterator=True) as reader:
 .....:    chunk1 = reader.read(5)
 .....:    chunk2 = reader.read(5)
 .....: 

df = pd.read_sas("sas_data.sas7bdat") 

def do_something(chunk):
    pass

with pd.read_sas("sas_xport.xpt", chunk=100000) as rdr:
    for chunk in rdr:
        do_something(chunk) 

df = pd.read_spss("spss_data.sav") 

df = pd.read_spss(
    "spss_data.sav",
    usecols=["foo", "bar"],
    convert_categoricals=False,
) 

In [1]: sz = 1000000
In [2]: df = pd.DataFrame({'A': np.random.randn(sz), 'B': [1] * sz})

In [3]: df.info()
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 1000000 entries, 0 to 999999
Data columns (total 2 columns):
A    1000000 non-null float64
B    1000000 non-null int64
dtypes: float64(1), int64(1)
memory usage: 15.3 MB 

import numpy as np

import os

sz = 1000000
df = pd.DataFrame({"A": np.random.randn(sz), "B": [1] * sz})

sz = 1000000
np.random.seed(42)
df = pd.DataFrame({"A": np.random.randn(sz), "B": [1] * sz})

def test_sql_write(df):
    if os.path.exists("test.sql"):
        os.remove("test.sql")
    sql_db = sqlite3.connect("test.sql")
    df.to_sql(name="test_table", con=sql_db)
    sql_db.close()

def test_sql_read():
    sql_db = sqlite3.connect("test.sql")
    pd.read_sql_query("select * from test_table", sql_db)
    sql_db.close()

def test_hdf_fixed_write(df):
    df.to_hdf("test_fixed.hdf", key="test", mode="w")

def test_hdf_fixed_read():
    pd.read_hdf("test_fixed.hdf", "test")

def test_hdf_fixed_write_compress(df):
    df.to_hdf("test_fixed_compress.hdf", key="test", mode="w", complib="blosc")

def test_hdf_fixed_read_compress():
    pd.read_hdf("test_fixed_compress.hdf", "test")

def test_hdf_table_write(df):
    df.to_hdf("test_table.hdf", key="test", mode="w", format="table")

def test_hdf_table_read():
    pd.read_hdf("test_table.hdf", "test")

def test_hdf_table_write_compress(df):
    df.to_hdf(
        "test_table_compress.hdf", key="test", mode="w", complib="blosc", format="table"
    )

def test_hdf_table_read_compress():
    pd.read_hdf("test_table_compress.hdf", "test")

def test_csv_write(df):
    df.to_csv("test.csv", mode="w")

def test_csv_read():
    pd.read_csv("test.csv", index_col=0)

def test_feather_write(df):
    df.to_feather("test.feather")

def test_feather_read():
    pd.read_feather("test.feather")

def test_pickle_write(df):
    df.to_pickle("test.pkl")

def test_pickle_read():
    pd.read_pickle("test.pkl")

def test_pickle_write_compress(df):
    df.to_pickle("test.pkl.compress", compression="xz")

def test_pickle_read_compress():
    pd.read_pickle("test.pkl.compress", compression="xz")

def test_parquet_write(df):
    df.to_parquet("test.parquet")

def test_parquet_read():
    pd.read_parquet("test.parquet") 

In [4]: %timeit test_sql_write(df)
3.29 s ± 43.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [5]: %timeit test_hdf_fixed_write(df)
19.4 ms ± 560 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [6]: %timeit test_hdf_fixed_write_compress(df)
19.6 ms ± 308 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [7]: %timeit test_hdf_table_write(df)
449 ms ± 5.61 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [8]: %timeit test_hdf_table_write_compress(df)
448 ms ± 11.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [9]: %timeit test_csv_write(df)
3.66 s ± 26.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [10]: %timeit test_feather_write(df)
9.75 ms ± 117 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [11]: %timeit test_pickle_write(df)
30.1 ms ± 229 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [12]: %timeit test_pickle_write_compress(df)
4.29 s ± 15.9 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [13]: %timeit test_parquet_write(df)
67.6 ms ± 706 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) 

In [14]: %timeit test_sql_read()
1.77 s ± 17.7 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [15]: %timeit test_hdf_fixed_read()
19.4 ms ± 436 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [16]: %timeit test_hdf_fixed_read_compress()
19.5 ms ± 222 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [17]: %timeit test_hdf_table_read()
38.6 ms ± 857 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [18]: %timeit test_hdf_table_read_compress()
38.8 ms ± 1.49 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [19]: %timeit test_csv_read()
452 ms ± 9.04 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [20]: %timeit test_feather_read()
12.4 ms ± 99.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [21]: %timeit test_pickle_read()
18.4 ms ± 191 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [22]: %timeit test_pickle_read_compress()
915 ms ± 7.48 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

In [23]: %timeit test_parquet_read()
24.4 ms ± 146 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) 

29519500 Oct 10 06:45 test.csv
16000248 Oct 10 06:45 test.feather
8281983  Oct 10 06:49 test.parquet
16000857 Oct 10 06:47 test.pkl
7552144  Oct 10 06:48 test.pkl.compress
34816000 Oct 10 06:42 test.sql
24009288 Oct 10 06:43 test_fixed.hdf
24009288 Oct 10 06:43 test_fixed_compress.hdf
24458940 Oct 10 06:44 test_table.hdf
24458940 Oct 10 06:44 test_table_compress.hdf 
```## CSV 和文本文件

用于读取文本文件（也称为平面文件）的主要函数是`read_csv()`。查看 cookbook 以获取一些高级策略。

### 解析选项

`read_csv()`接受以下常见参数：

#### 基本

filepath_or_buffervarious

要么是文件路径（[`str`](https://docs.python.org/3/library/stdtypes.html#str "(在 Python v3.12 中)")），[`pathlib.Path`](https://docs.python.org/3/library/pathlib.html#pathlib.Path "(在 Python v3.12 中)")，或`py:py._path.local.LocalPath`），URL（包括 http，ftp 和 S3 位置），或具有`read()`方法的任何对象（例如打开的文件或[`StringIO`](https://docs.python.org/3/library/io.html#io.StringIO "(在 Python v3.12 中)")）。

sepstr，默认为`','`用于`read_csv()`，`\t`用于`read_table()`

要使用的分隔符。如果 sep 为`None`，C 引擎无法自动检测分隔符，但 Python 解析引擎可以，这意味着将使用后者，并通过 Python 的内置 sniffer 工具[`csv.Sniffer`](https://docs.python.org/3/library/csv.html#csv.Sniffer "(in Python v3.12)")自动检测分隔符。此外，长于 1 个字符且不同于`'\s+'`的分隔符将被解释为正则表达式，并且还会强制使用 Python 解析引擎。请注意，正则表达式分隔符容易忽略带引号的数据。正则表达式示例：`'\\r\\t'`。

分隔符字符串，默认为`None`

sep 的替代参数名称。

delim_whitespace 布尔值，默认为 False

指定是否使用空格（例如`' '`或`'\t'`）作为分隔符。等效于设置`sep='\s+'`。如果将此选项设置为`True`，则不应为`delimiter`参数传递任何内容。

#### 列和索引位置及名称

headerint 或 int 列表，默认为`'infer'`

用作列名和数据起始位置的行号。默认行为是推断列名：如果没有传递名称，则行为与`header=0`相同，并且列名从文件的第一行推断出来，如果显式传递了列名，则行为与`header=None`相同。显式传递`header=0`以能够替换现有名称。

表头可以是指定列的 MultiIndex 的行位置列表，例如`[0,1,3]`。未指定的中间行将被跳过（例如，在此示例中跳过了 2）。请注意，如果`skip_blank_lines=True`，此参数将忽略注释行和空行，因此`header=0`表示数据的第一行而不是文件的第一行。

names 数组样式，默认为`None`

要使用的列名列表。如果文件不包含表头行，则应明确传递`header=None`。不允许在此列表中存在重复项。

index_colint、str、int/str 序列或 False，可选，默认为`None`

用作`DataFrame`行标签的列，可以是字符串名称或列索引。如果给定 int/str 序列，则使用 MultiIndex。

注意

`index_col=False`可用于强制 pandas*不*将第一列用作索引，例如当您有一个每行末尾都有分隔符的格式不正确的文件时。

`None`的默认值指示 pandas 进行猜测。如果列头行中的字段数等于数据文件主体中的字段数，则使用默认索引。如果大于，则使用前几列作为索引，以使数据主体中的字段数等于列头中的字段数。

表头后的第一行用于确定列数，这些列将进入索引。如果后续行的列数少于第一行，则用`NaN`填充。

可通过 `usecols` 避免这种情况。这确保列按原样采取，并且尾随数据被忽略。

usecols 类似列表或可调用函数，默认为 `None`。

返回列的子集。如果类似列表，所有元素必须是位置的（即整数索引到文档列）或与用户在 `names` 中提供的列名对应的字符串，或从文档标题行中推断出的列名。如果给定了 `names`，则不考虑文档标题行。例如，一个有效的类似列表 `usecols` 参数可以是 `[0, 1, 2]` 或 `['foo', 'bar', 'baz']`。

元素顺序被忽略，因此 `usecols=[0, 1]` 与 `[1, 0]` 相同。要从保留元素顺序的 `data` 实例化 DataFrame，请使用 `pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]` 以 `['foo', 'bar']` 顺序的列或 `pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]` 以 `['bar', 'foo']` 顺序。

如果可调用，则将对列名评估可调用函数，返回可调用函数评估为 True 的列名：

```py
In [1]: import pandas as pd

In [2]: from io import StringIO

In [3]: data = "col1,col2,col3\na,b,1\na,b,2\nc,d,3"

In [4]: pd.read_csv(StringIO(data))
Out[4]: 
 col1 col2  col3
0    a    b     1
1    a    b     2
2    c    d     3

In [5]: pd.read_csv(StringIO(data), usecols=lambda x: x.upper() in ["COL1", "COL3"])
Out[5]: 
 col1  col3
0    a     1
1    a     2
2    c     3 

In [6]: data = "col1,col2,col3\na,b,1\na,b,2\nc,d,3"

In [7]: pd.read_csv(StringIO(data))
Out[7]: 
 col1 col2  col3
0    a    b     1
1    a    b     2
2    c    d     3

In [8]: pd.read_csv(StringIO(data), skiprows=lambda x: x % 2 != 0)
Out[8]: 
 col1 col2  col3
0    a    b     2 

In [9]: import numpy as np

In [10]: data = "a,b,c,d\n1,2,3,4\n5,6,7,8\n9,10,11"

In [11]: print(data)
a,b,c,d
1,2,3,4
5,6,7,8
9,10,11

In [12]: df = pd.read_csv(StringIO(data), dtype=object)

In [13]: df
Out[13]: 
 a   b   c    d
0  1   2   3    4
1  5   6   7    8
2  9  10  11  NaN

In [14]: df["a"][0]
Out[14]: '1'

In [15]: df = pd.read_csv(StringIO(data), dtype={"b": object, "c": np.float64, "d": "Int64"})

In [16]: df.dtypes
Out[16]: 
a      int64
b     object
c    float64
d      Int64
dtype: object 

In [17]: data = "col_1\n1\n2\n'A'\n4.22"

In [18]: df = pd.read_csv(StringIO(data), converters={"col_1": str})

In [19]: df
Out[19]: 
 col_1
0     1
1     2
2   'A'
3  4.22

In [20]: df["col_1"].apply(type).value_counts()
Out[20]: 
col_1
<class 'str'>    4
Name: count, dtype: int64 

In [21]: df2 = pd.read_csv(StringIO(data))

In [22]: df2["col_1"] = pd.to_numeric(df2["col_1"], errors="coerce")

In [23]: df2
Out[23]: 
 col_1
0   1.00
1   2.00
2    NaN
3   4.22

In [24]: df2["col_1"].apply(type).value_counts()
Out[24]: 
col_1
<class 'float'>    4
Name: count, dtype: int64 

In [25]: col_1 = list(range(500000)) + ["a", "b"] + list(range(500000))

In [26]: df = pd.DataFrame({"col_1": col_1})

In [27]: df.to_csv("foo.csv")

In [28]: mixed_df = pd.read_csv("foo.csv")

In [29]: mixed_df["col_1"].apply(type).value_counts()
Out[29]: 
col_1
<class 'int'>    737858
<class 'str'>    262144
Name: count, dtype: int64

In [30]: mixed_df["col_1"].dtype
Out[30]: dtype('O') 

In [31]: data = """a,b,c,d,e,f,g,h,i,j
 ....: 1,2.5,True,a,,,,,12-31-2019,
 ....: 3,4.5,False,b,6,7.5,True,a,12-31-2019,
 ....: """
 ....: 

In [32]: df = pd.read_csv(StringIO(data), dtype_backend="numpy_nullable", parse_dates=["i"])

In [33]: df
Out[33]: 
 a    b      c  d     e     f     g     h          i     j
0  1  2.5   True  a  <NA>  <NA>  <NA>  <NA> 2019-12-31  <NA>
1  3  4.5  False  b     6   7.5  True     a 2019-12-31  <NA>

In [34]: df.dtypes
Out[34]: 
a             Int64
b           Float64
c           boolean
d    string[python]
e             Int64
f           Float64
g           boolean
h    string[python]
i    datetime64[ns]
j             Int64
dtype: object 
```### 指定分类 dtype

可以通过指定`dtype='category'`或`dtype=CategoricalDtype(categories, ordered)`直接解析`Categorical`列。

```py
In [35]: data = "col1,col2,col3\na,b,1\na,b,2\nc,d,3"

In [36]: pd.read_csv(StringIO(data))
Out[36]: 
 col1 col2  col3
0    a    b     1
1    a    b     2
2    c    d     3

In [37]: pd.read_csv(StringIO(data)).dtypes
Out[37]: 
col1    object
col2    object
col3     int64
dtype: object

In [38]: pd.read_csv(StringIO(data), dtype="category").dtypes
Out[38]: 
col1    category
col2    category
col3    category
dtype: object 

In [39]: pd.read_csv(StringIO(data), dtype={"col1": "category"}).dtypes
Out[39]: 
col1    category
col2      object
col3       int64
dtype: object 

In [40]: from pandas.api.types import CategoricalDtype

In [41]: dtype = CategoricalDtype(["d", "c", "b", "a"], ordered=True)

In [42]: pd.read_csv(StringIO(data), dtype={"col1": dtype}).dtypes
Out[42]: 
col1    category
col2      object
col3       int64
dtype: object 

In [43]: dtype = CategoricalDtype(["a", "b", "d"])  # No 'c'

In [44]: pd.read_csv(StringIO(data), dtype={"col1": dtype}).col1
Out[44]: 
0      a
1      a
2    NaN
Name: col1, dtype: category
Categories (3, object): ['a', 'b', 'd'] 

In [45]: df = pd.read_csv(StringIO(data), dtype="category")

In [46]: df.dtypes
Out[46]: 
col1    category
col2    category
col3    category
dtype: object

In [47]: df["col3"]
Out[47]: 
0    1
1    2
2    3
Name: col3, dtype: category
Categories (3, object): ['1', '2', '3']

In [48]: new_categories = pd.to_numeric(df["col3"].cat.categories)

In [49]: df["col3"] = df["col3"].cat.rename_categories(new_categories)

In [50]: df["col3"]
Out[50]: 
0    1
1    2
2    3
Name: col3, dtype: category
Categories (3, int64): [1, 2, 3] 

In [51]: data = "a,b,c\n1,2,3\n4,5,6\n7,8,9"

In [52]: print(data)
a,b,c
1,2,3
4,5,6
7,8,9

In [53]: pd.read_csv(StringIO(data))
Out[53]: 
 a  b  c
0  1  2  3
1  4  5  6
2  7  8  9 

In [54]: print(data)
a,b,c
1,2,3
4,5,6
7,8,9

In [55]: pd.read_csv(StringIO(data), names=["foo", "bar", "baz"], header=0)
Out[55]: 
 foo  bar  baz
0    1    2    3
1    4    5    6
2    7    8    9

In [56]: pd.read_csv(StringIO(data), names=["foo", "bar", "baz"], header=None)
Out[56]: 
 foo bar baz
0   a   b   c
1   1   2   3
2   4   5   6
3   7   8   9 

In [57]: data = "skip this skip it\na,b,c\n1,2,3\n4,5,6\n7,8,9"

In [58]: pd.read_csv(StringIO(data), header=1)
Out[58]: 
 a  b  c
0  1  2  3
1  4  5  6
2  7  8  9 

In [59]: data = "a,b,a\n0,1,2\n3,4,5"

In [60]: pd.read_csv(StringIO(data))
Out[60]: 
 a  b  a.1
0  0  1    2
1  3  4    5 

In [61]: data = "a,b,c,d\n1,2,3,foo\n4,5,6,bar\n7,8,9,baz"

In [62]: pd.read_csv(StringIO(data))
Out[62]: 
 a  b  c    d
0  1  2  3  foo
1  4  5  6  bar
2  7  8  9  baz

In [63]: pd.read_csv(StringIO(data), usecols=["b", "d"])
Out[63]: 
 b    d
0  2  foo
1  5  bar
2  8  baz

In [64]: pd.read_csv(StringIO(data), usecols=[0, 2, 3])
Out[64]: 
 a  c    d
0  1  3  foo
1  4  6  bar
2  7  9  baz

In [65]: pd.read_csv(StringIO(data), usecols=lambda x: x.upper() in ["A", "C"])
Out[65]: 
 a  c
0  1  3
1  4  6
2  7  9 

In [66]: pd.read_csv(StringIO(data), usecols=lambda x: x not in ["a", "c"])
Out[66]: 
 b    d
0  2  foo
1  5  bar
2  8  baz 

In [67]: data = "\na,b,c\n  \n# commented line\n1,2,3\n\n4,5,6"

In [68]: print(data)

a,b,c

# commented line
1,2,3

4,5,6

In [69]: pd.read_csv(StringIO(data), comment="#")
Out[69]: 
 a  b  c
0  1  2  3
1  4  5  6 

In [70]: data = "a,b,c\n\n1,2,3\n\n\n4,5,6"

In [71]: pd.read_csv(StringIO(data), skip_blank_lines=False)
Out[71]: 
 a    b    c
0  NaN  NaN  NaN
1  1.0  2.0  3.0
2  NaN  NaN  NaN
3  NaN  NaN  NaN
4  4.0  5.0  6.0 

In [72]: data = "#comment\na,b,c\nA,B,C\n1,2,3"

In [73]: pd.read_csv(StringIO(data), comment="#", header=1)
Out[73]: 
 A  B  C
0  1  2  3

In [74]: data = "A,B,C\n#comment\na,b,c\n1,2,3"

In [75]: pd.read_csv(StringIO(data), comment="#", skiprows=2)
Out[75]: 
 a  b  c
0  1  2  3 

In [76]: data = (
 ....:    "# empty\n"
 ....:    "# second empty line\n"
 ....:    "# third emptyline\n"
 ....:    "X,Y,Z\n"
 ....:    "1,2,3\n"
 ....:    "A,B,C\n"
 ....:    "1,2.,4.\n"
 ....:    "5.,NaN,10.0\n"
 ....: )
 ....: 

In [77]: print(data)
# empty
# second empty line
# third emptyline
X,Y,Z
1,2,3
A,B,C
1,2.,4.
5.,NaN,10.0

In [78]: pd.read_csv(StringIO(data), comment="#", skiprows=4, header=1)
Out[78]: 
 A    B     C
0  1.0  2.0   4.0
1  5.0  NaN  10.0 
```#### 注释

有时文件中可能包含注释或元数据：

```py
In [79]: data = (
 ....:    "ID,level,category\n"
 ....:    "Patient1,123000,x # really unpleasant\n"
 ....:    "Patient2,23000,y # wouldn't take his medicine\n"
 ....:    "Patient3,1234018,z # awesome"
 ....: )
 ....: 

In [80]: with open("tmp.csv", "w") as fh:
 ....:    fh.write(data)
 ....: 

In [81]: print(open("tmp.csv").read())
ID,level,category
Patient1,123000,x # really unpleasant
Patient2,23000,y # wouldn't take his medicine
Patient3,1234018,z # awesome 

In [82]: df = pd.read_csv("tmp.csv")

In [83]: df
Out[83]: 
 ID    level                        category
0  Patient1   123000           x # really unpleasant
1  Patient2    23000  y # wouldn't take his medicine
2  Patient3  1234018                     z # awesome 

In [84]: df = pd.read_csv("tmp.csv", comment="#")

In [85]: df
Out[85]: 
 ID    level category
0  Patient1   123000       x 
1  Patient2    23000       y 
2  Patient3  1234018       z 
```### 处理 Unicode 数据

应该使用`encoding`参数来处理编码的 Unicode 数据，这将导致字节字符串在结果中被解码为 Unicode：

```py
In [86]: from io import BytesIO

In [87]: data = b"word,length\n" b"Tr\xc3\xa4umen,7\n" b"Gr\xc3\xbc\xc3\x9fe,5"

In [88]: data = data.decode("utf8").encode("latin-1")

In [89]: df = pd.read_csv(BytesIO(data), encoding="latin-1")

In [90]: df
Out[90]: 
 word  length
0  Träumen       7
1    Grüße       5

In [91]: df["word"][1]
Out[91]: 'Grüße' 

In [92]: data = "a,b,c\n4,apple,bat,5.7\n8,orange,cow,10"

In [93]: pd.read_csv(StringIO(data))
Out[93]: 
 a    b     c
4   apple  bat   5.7
8  orange  cow  10.0 

In [94]: data = "index,a,b,c\n4,apple,bat,5.7\n8,orange,cow,10"

In [95]: pd.read_csv(StringIO(data), index_col=0)
Out[95]: 
 a    b     c
index 
4       apple  bat   5.7
8      orange  cow  10.0 

In [96]: data = "a,b,c\n4,apple,bat,\n8,orange,cow,"

In [97]: print(data)
a,b,c
4,apple,bat,
8,orange,cow,

In [98]: pd.read_csv(StringIO(data))
Out[98]: 
 a    b   c
4   apple  bat NaN
8  orange  cow NaN

In [99]: pd.read_csv(StringIO(data), index_col=False)
Out[99]: 
 a       b    c
0  4   apple  bat
1  8  orange  cow 

In [100]: data = "a,b,c\n4,apple,bat,\n8,orange,cow,"

In [101]: print(data)
a,b,c
4,apple,bat,
8,orange,cow,

In [102]: pd.read_csv(StringIO(data), usecols=["b", "c"])
Out[102]: 
 b   c
4  bat NaN
8  cow NaN

In [103]: pd.read_csv(StringIO(data), usecols=["b", "c"], index_col=0)
Out[103]: 
 b   c
4  bat NaN
8  cow NaN 
```### 日期处理

#### 指定日期列

为了更好地处理日期时间数据，`read_csv()`使用关键字参数`parse_dates`和`date_format`允许用户指定各种列和日期/时间格式将输入文本数据转换为`datetime`对象。

最简单的情况是只需传入`parse_dates=True`：

```py
In [104]: with open("foo.csv", mode="w") as f:
 .....:    f.write("date,A,B,C\n20090101,a,1,2\n20090102,b,3,4\n20090103,c,4,5")
 .....: 

# Use a column as an index, and parse it as dates.
In [105]: df = pd.read_csv("foo.csv", index_col=0, parse_dates=True)

In [106]: df
Out[106]: 
 A  B  C
date 
2009-01-01  a  1  2
2009-01-02  b  3  4
2009-01-03  c  4  5

# These are Python datetime objects
In [107]: df.index
Out[107]: DatetimeIndex(['2009-01-01', '2009-01-02', '2009-01-03'], dtype='datetime64[ns]', name='date', freq=None) 

In [108]: data = (
 .....:    "KORD,19990127, 19:00:00, 18:56:00, 0.8100\n"
 .....:    "KORD,19990127, 20:00:00, 19:56:00, 0.0100\n"
 .....:    "KORD,19990127, 21:00:00, 20:56:00, -0.5900\n"
 .....:    "KORD,19990127, 21:00:00, 21:18:00, -0.9900\n"
 .....:    "KORD,19990127, 22:00:00, 21:56:00, -0.5900\n"
 .....:    "KORD,19990127, 23:00:00, 22:56:00, -0.5900"
 .....: )
 .....: 

In [109]: with open("tmp.csv", "w") as fh:
 .....:    fh.write(data)
 .....: 

In [110]: df = pd.read_csv("tmp.csv", header=None, parse_dates=[[1, 2], [1, 3]])

In [111]: df
Out[111]: 
 1_2                 1_3     0     4
0 1999-01-27 19:00:00 1999-01-27 18:56:00  KORD  0.81
1 1999-01-27 20:00:00 1999-01-27 19:56:00  KORD  0.01
2 1999-01-27 21:00:00 1999-01-27 20:56:00  KORD -0.59
3 1999-01-27 21:00:00 1999-01-27 21:18:00  KORD -0.99
4 1999-01-27 22:00:00 1999-01-27 21:56:00  KORD -0.59
5 1999-01-27 23:00:00 1999-01-27 22:56:00  KORD -0.59 

In [112]: df = pd.read_csv(
 .....:    "tmp.csv", header=None, parse_dates=[[1, 2], [1, 3]], keep_date_col=True
 .....: )
 .....: 

In [113]: df
Out[113]: 
 1_2                 1_3     0  ...          2          3     4
0 1999-01-27 19:00:00 1999-01-27 18:56:00  KORD  ...   19:00:00   18:56:00  0.81
1 1999-01-27 20:00:00 1999-01-27 19:56:00  KORD  ...   20:00:00   19:56:00  0.01
2 1999-01-27 21:00:00 1999-01-27 20:56:00  KORD  ...   21:00:00   20:56:00 -0.59
3 1999-01-27 21:00:00 1999-01-27 21:18:00  KORD  ...   21:00:00   21:18:00 -0.99
4 1999-01-27 22:00:00 1999-01-27 21:56:00  KORD  ...   22:00:00   21:56:00 -0.59
5 1999-01-27 23:00:00 1999-01-27 22:56:00  KORD  ...   23:00:00   22:56:00 -0.59

[6 rows x 7 columns] 

In [114]: date_spec = {"nominal": [1, 2], "actual": [1, 3]}

In [115]: df = pd.read_csv("tmp.csv", header=None, parse_dates=date_spec)

In [116]: df
Out[116]: 
 nominal              actual     0     4
0 1999-01-27 19:00:00 1999-01-27 18:56:00  KORD  0.81
1 1999-01-27 20:00:00 1999-01-27 19:56:00  KORD  0.01
2 1999-01-27 21:00:00 1999-01-27 20:56:00  KORD -0.59
3 1999-01-27 21:00:00 1999-01-27 21:18:00  KORD -0.99
4 1999-01-27 22:00:00 1999-01-27 21:56:00  KORD -0.59
5 1999-01-27 23:00:00 1999-01-27 22:56:00  KORD -0.59 

In [117]: date_spec = {"nominal": [1, 2], "actual": [1, 3]}

In [118]: df = pd.read_csv(
 .....:    "tmp.csv", header=None, parse_dates=date_spec, index_col=0
 .....: )  # index is the nominal column
 .....: 

In [119]: df
Out[119]: 
 actual     0     4
nominal 
1999-01-27 19:00:00 1999-01-27 18:56:00  KORD  0.81
1999-01-27 20:00:00 1999-01-27 19:56:00  KORD  0.01
1999-01-27 21:00:00 1999-01-27 20:56:00  KORD -0.59
1999-01-27 21:00:00 1999-01-27 21:18:00  KORD -0.99
1999-01-27 22:00:00 1999-01-27 21:56:00  KORD -0.59
1999-01-27 23:00:00 1999-01-27 22:56:00  KORD -0.59 

In [120]: content = """\
 .....: a
 .....: 2000-01-01T00:00:00+05:00
 .....: 2000-01-01T00:00:00+06:00"""
 .....: 

In [121]: df = pd.read_csv(StringIO(content))

In [122]: df["a"] = pd.to_datetime(df["a"], utc=True)

In [123]: df["a"]
Out[123]: 
0   1999-12-31 19:00:00+00:00
1   1999-12-31 18:00:00+00:00
Name: a, dtype: datetime64[ns, UTC] 
```#### 推断日期时间格式

这里有一些可以猜测的日期时间字符串示例（均表示 2011 年 12 月 30 日 00:00:00）：

+   “20111230”

+   “2011/12/30”

+   “20111230 00:00:00”

+   “12/30/2011 00:00:00”

+   “30/Dec/2011 00:00:00”

+   “30/December/2011 00:00:00”

请注意，格式推断对`dayfirst`很敏感。使用`dayfirst=True`，它会猜测“01/12/2011”为 12 月 1 日。使用`dayfirst=False`（默认），它会猜测“01/12/2011”为 1 月 12 日。

如果尝试解析日期字符串列，pandas 将尝试从第一个非 NaN 元素猜测格式，然后使用该格式解析列的其余部分。如果 pandas 无法猜测格式（例如，如果您的第一个字符串是`'01 December US/Pacific 2000'`），则会发出警告，并且每行将通过`dateutil.parser.parse`单独解析。解析日期的最安全方式是显式设置`format=`。

```py
In [124]: df = pd.read_csv(
 .....:    "foo.csv",
 .....:    index_col=0,
 .....:    parse_dates=True,
 .....: )
 .....: 

In [125]: df
Out[125]: 
 A  B  C
date 
2009-01-01  a  1  2
2009-01-02  b  3  4
2009-01-03  c  4  5 

In [126]: data = StringIO("date\n12 Jan 2000\n2000-01-13\n")

In [127]: df = pd.read_csv(data)

In [128]: df['date'] = pd.to_datetime(df['date'], format='mixed')

In [129]: df
Out[129]: 
 date
0 2000-01-12
1 2000-01-13 

In [130]: data = StringIO("date\n2020-01-01\n2020-01-01 03:00\n")

In [131]: df = pd.read_csv(data)

In [132]: df['date'] = pd.to_datetime(df['date'], format='ISO8601')

In [133]: df
Out[133]: 
 date
0 2020-01-01 00:00:00
1 2020-01-01 03:00:00 

In [134]: data = "date,value,cat\n1/6/2000,5,a\n2/6/2000,10,b\n3/6/2000,15,c"

In [135]: print(data)
date,value,cat
1/6/2000,5,a
2/6/2000,10,b
3/6/2000,15,c

In [136]: with open("tmp.csv", "w") as fh:
 .....:    fh.write(data)
 .....: 

In [137]: pd.read_csv("tmp.csv", parse_dates=[0])
Out[137]: 
 date  value cat
0 2000-01-06      5   a
1 2000-02-06     10   b
2 2000-03-06     15   c

In [138]: pd.read_csv("tmp.csv", dayfirst=True, parse_dates=[0])
Out[138]: 
 date  value cat
0 2000-06-01      5   a
1 2000-06-02     10   b
2 2000-06-03     15   c 

In [139]: import io

In [140]: data = pd.DataFrame([0, 1, 2])

In [141]: buffer = io.BytesIO()

In [142]: data.to_csv(buffer, encoding="utf-8", compression="gzip") 
```### 指定浮点转换方法

可以通过指定 `float_precision` 参数来使用特定的浮点数转换器在 C 引擎解析时。选项有普通转换器、高精度转换器和往返转换器（保证在写入文件后循环的值）。例如：

```py
In [143]: val = "0.3066101993807095471566981359501369297504425048828125"

In [144]: data = "a,b,c\n1,2,{0}".format(val)

In [145]: abs(
 .....:    pd.read_csv(
 .....:        StringIO(data),
 .....:        engine="c",
 .....:        float_precision=None,
 .....:    )["c"][0] - float(val)
 .....: )
 .....: 
Out[145]: 5.551115123125783e-17

In [146]: abs(
 .....:    pd.read_csv(
 .....:        StringIO(data),
 .....:        engine="c",
 .....:        float_precision="high",
 .....:    )["c"][0] - float(val)
 .....: )
 .....: 
Out[146]: 5.551115123125783e-17

In [147]: abs(
 .....:    pd.read_csv(StringIO(data), engine="c", float_precision="round_trip")["c"][0]
 .....:    - float(val)
 .....: )
 .....: 
Out[147]: 0.0 
```### 千位分隔符

对于以千位分隔符编写的大数字，您可以将 `thousands` 关键字设置为长度为 1 的字符串，以便正确解析整数：

默认情况下，带有千位分隔符的数字将被解析为字符串：

```py
In [148]: data = (
 .....:    "ID|level|category\n"
 .....:    "Patient1|123,000|x\n"
 .....:    "Patient2|23,000|y\n"
 .....:    "Patient3|1,234,018|z"
 .....: )
 .....: 

In [149]: with open("tmp.csv", "w") as fh:
 .....:    fh.write(data)
 .....: 

In [150]: df = pd.read_csv("tmp.csv", sep="|")

In [151]: df
Out[151]: 
 ID      level category
0  Patient1    123,000        x
1  Patient2     23,000        y
2  Patient3  1,234,018        z

In [152]: df.level.dtype
Out[152]: dtype('O') 

In [153]: df = pd.read_csv("tmp.csv", sep="|", thousands=",")

In [154]: df
Out[154]: 
 ID    level category
0  Patient1   123000        x
1  Patient2    23000        y
2  Patient3  1234018        z

In [155]: df.level.dtype
Out[155]: dtype('int64') 
```### NA 值

要控制哪些值被解析为缺失值（用 `NaN` 表示），请在 `na_values` 中指定一个字符串。如果您指定了一个字符串列表，那么其中的所有值都将被视为缺失值。如果您指定了一个数字（一个 `float`，比如 `5.0` 或一个 `integer`，比如 `5`），则相应的等价值也将暗示一个缺失值（在这种情况下，实际上 `[5.0, 5]` 被认为是 `NaN`）。

要完全覆盖默认被识别为缺失的值，请指定 `keep_default_na=False`。

默认的 `NaN` 被识别的值为 `['-1.#IND', '1.#QNAN', '1.#IND', '-1.#QNAN', '#N/A N/A', '#N/A', 'N/A', 'n/a', 'NA', '<NA>', '#NA', 'NULL', 'null', 'NaN', '-NaN', 'nan', '-nan', 'None', '']`。

让我们考虑一些例子：

```py
pd.read_csv("path_to_file.csv", na_values=[5]) 

pd.read_csv("path_to_file.csv", keep_default_na=False, na_values=[""]) 

pd.read_csv("path_to_file.csv", keep_default_na=False, na_values=["NA", "0"]) 

pd.read_csv("path_to_file.csv", na_values=["Nope"]) 

In [156]: data = "a,b,c\n1,Yes,2\n3,No,4"

In [157]: print(data)
a,b,c
1,Yes,2
3,No,4

In [158]: pd.read_csv(StringIO(data))
Out[158]: 
 a    b  c
0  1  Yes  2
1  3   No  4

In [159]: pd.read_csv(StringIO(data), true_values=["Yes"], false_values=["No"])
Out[159]: 
 a      b  c
0  1   True  2
1  3  False  4 
```### 处理“坏”行

一些文件可能存在字段过少或过多的格式不正确的行。字段过少的行将在尾部字段中填充 NA 值。字段过多的行将默认引发错误：

```py
In [160]: data = "a,b,c\n1,2,3\n4,5,6,7\n8,9,10"

In [161]: pd.read_csv(StringIO(data))
---------------------------------------------------------------------------
ParserError  Traceback (most recent call last)
Cell In[161], line 1
----> 1 pd.read_csv(StringIO(data))

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:1026, in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)
  1013 kwds_defaults = _refine_defaults_read(
  1014     dialect,
  1015     delimiter,
   (...)
  1022     dtype_backend=dtype_backend,
  1023 )
  1024 kwds.update(kwds_defaults)
-> 1026 return _read(filepath_or_buffer, kwds)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:626, in _read(filepath_or_buffer, kwds)
  623     return parser
  625 with parser:
--> 626     return parser.read(nrows)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:1923, in TextFileReader.read(self, nrows)
  1916 nrows = validate_integer("nrows", nrows)
  1917 try:
  1918     # error: "ParserBase" has no attribute "read"
  1919     (
  1920         index,
  1921         columns,
  1922         col_dict,
-> 1923     ) = self._engine.read(  # type: ignore[attr-defined]
  1924         nrows
  1925     )
  1926 except Exception:
  1927     self.close()

File ~/work/pandas/pandas/pandas/io/parsers/c_parser_wrapper.py:234, in CParserWrapper.read(self, nrows)
  232 try:
  233     if self.low_memory:
--> 234         chunks = self._reader.read_low_memory(nrows)
  235         # destructive to chunks
  236         data = _concatenate_chunks(chunks)

File parsers.pyx:838, in pandas._libs.parsers.TextReader.read_low_memory()

File parsers.pyx:905, in pandas._libs.parsers.TextReader._read_rows()

File parsers.pyx:874, in pandas._libs.parsers.TextReader._tokenize_rows()

File parsers.pyx:891, in pandas._libs.parsers.TextReader._check_tokenize_status()

File parsers.pyx:2061, in pandas._libs.parsers.raise_parser_error()

ParserError: Error tokenizing data. C error: Expected 3 fields in line 3, saw 4 

In [162]: data = "a,b,c\n1,2,3\n4,5,6,7\n8,9,10"

In [163]: pd.read_csv(StringIO(data), on_bad_lines="skip")
Out[163]: 
 a  b   c
0  1  2   3
1  8  9  10 

In [164]: external_list = []

In [165]: def bad_lines_func(line):
 .....:    external_list.append(line)
 .....:    return line[-3:]
 .....: 

In [166]: external_list
Out[166]: [] 

In [167]: bad_lines_func = lambda line: print(line)

In [168]: data = 'name,type\nname a,a is of type a\nname b,"b\" is of type b"'

In [169]: data
Out[169]: 'name,type\nname a,a is of type a\nname b,"b" is of type b"'

In [170]: pd.read_csv(StringIO(data), on_bad_lines=bad_lines_func, engine="python")
Out[170]: 
 name            type
0  name a  a is of type a 

In [171]: pd.read_csv(StringIO(data), usecols=[0, 1, 2])
---------------------------------------------------------------------------
ValueError  Traceback (most recent call last)
Cell In[171], line 1
----> 1 pd.read_csv(StringIO(data), usecols=[0, 1, 2])

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:1026, in read_csv(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)
  1013 kwds_defaults = _refine_defaults_read(
  1014     dialect,
  1015     delimiter,
   (...)
  1022     dtype_backend=dtype_backend,
  1023 )
  1024 kwds.update(kwds_defaults)
-> 1026 return _read(filepath_or_buffer, kwds)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:620, in _read(filepath_or_buffer, kwds)
  617 _validate_names(kwds.get("names", None))
  619 # Create the parser.
--> 620 parser = TextFileReader(filepath_or_buffer, **kwds)
  622 if chunksize or iterator:
  623     return parser

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:1620, in TextFileReader.__init__(self, f, engine, **kwds)
  1617     self.options["has_index_names"] = kwds["has_index_names"]
  1619 self.handles: IOHandles | None = None
-> 1620 self._engine = self._make_engine(f, self.engine)

File ~/work/pandas/pandas/pandas/io/parsers/readers.py:1898, in TextFileReader._make_engine(self, f, engine)
  1895     raise ValueError(msg)
  1897 try:
-> 1898     return mappingengine
  1899 except Exception:
  1900     if self.handles is not None:

File ~/work/pandas/pandas/pandas/io/parsers/c_parser_wrapper.py:155, in CParserWrapper.__init__(self, src, **kwds)
  152     # error: Cannot determine type of 'names'
  153     if len(self.names) < len(usecols):  # type: ignore[has-type]
  154         # error: Cannot determine type of 'names'
--> 155         self._validate_usecols_names(
  156             usecols,
  157             self.names,  # type: ignore[has-type]
  158         )
  160 # error: Cannot determine type of 'names'
  161 self._validate_parse_dates_presence(self.names)  # type: ignore[has-type]

File ~/work/pandas/pandas/pandas/io/parsers/base_parser.py:979, in ParserBase._validate_usecols_names(self, usecols, names)
  977 missing = [c for c in usecols if c not in names]
  978 if len(missing) > 0:
--> 979     raise ValueError(
  980         f"Usecols do not match columns, columns expected but not found: "
  981         f"{missing}"
  982     )
  984 return usecols

ValueError: Usecols do not match columns, columns expected but not found: [0, 1, 2] 

In [172]: pd.read_csv(StringIO(data), names=['a', 'b', 'c', 'd'])
Out[172]: 
 a                b   c   d
0    name             type NaN NaN
1  name a   a is of type a NaN NaN
2  name b  b is of type b" NaN NaN 
```### 方言

`dialect`关键字提供了更大的灵活性来指定文件格式。默认情况下使用 Excel 方言，但您可以指定方言名称或[`csv.Dialect`](https://docs.python.org/3/library/csv.html#csv.Dialect "(在 Python v3.12 中)")实例。

假设您的数据中有未封闭的引号：

```py
In [173]: data = "label1,label2,label3\n" 'index1,"a,c,e\n' "index2,b,d,f"

In [174]: print(data)
label1,label2,label3
index1,"a,c,e
index2,b,d,f 

In [175]: import csv

In [176]: dia = csv.excel()

In [177]: dia.quoting = csv.QUOTE_NONE

In [178]: pd.read_csv(StringIO(data), dialect=dia)
Out[178]: 
 label1 label2 label3
index1     "a      c      e
index2      b      d      f 

In [179]: data = "a,b,c~1,2,3~4,5,6"

In [180]: pd.read_csv(StringIO(data), lineterminator="~")
Out[180]: 
 a  b  c
0  1  2  3
1  4  5  6 

In [181]: data = "a, b, c\n1, 2, 3\n4, 5, 6"

In [182]: print(data)
a, b, c
1, 2, 3
4, 5, 6

In [183]: pd.read_csv(StringIO(data), skipinitialspace=True)
Out[183]: 
 a  b  c
0  1  2  3
1  4  5  6 

In [184]: data = 'a,b\n"hello, \\"Bob\\", nice to see you",5'

In [185]: print(data)
a,b
"hello, \"Bob\", nice to see you",5

In [186]: pd.read_csv(StringIO(data), escapechar="\\")
Out[186]: 
 a  b
0  hello, "Bob", nice to see you  5 
```### 具有固定宽度列的文件

虽然`read_csv()`用于读取分隔数据，`read_fwf()`函数用于处理具有已知和固定列宽的数据文件。`read_fwf`的函数参数与`read_csv`基本相同，但有两个额外参数，并且`delimiter`参数的使用方式不同：

+   `colspecs`：一个对给出每行固定宽度字段的范围的一半开放区间（即，[from, to[）的列表（元组）。字符串值‘infer’ 可以用于指示解析器尝试从数据的前 100 行检测列规格。如果未指定，默认行为是推断。

+   `widths`：一个字段宽度的列表，可用于代替‘colspecs’，如果间隔是连续的。

+   `delimiter`：在固定宽度文件中视为填充字符的字符。如果字段的填充字符不是空格（例如，‘~’），则可以用它来指定字段的填充字符。

考虑一个典型的固定宽度数据文件：

```py
In [187]: data1 = (
 .....:    "id8141    360.242940   149.910199   11950.7\n"
 .....:    "id1594    444.953632   166.985655   11788.4\n"
 .....:    "id1849    364.136849   183.628767   11806.2\n"
 .....:    "id1230    413.836124   184.375703   11916.8\n"
 .....:    "id1948    502.953953   173.237159   12468.3"
 .....: )
 .....: 

In [188]: with open("bar.csv", "w") as f:
 .....:    f.write(data1)
 .....: 

# Column specifications are a list of half-intervals
In [189]: colspecs = [(0, 6), (8, 20), (21, 33), (34, 43)]

In [190]: df = pd.read_fwf("bar.csv", colspecs=colspecs, header=None, index_col=0)

In [191]: df
Out[191]: 
 1           2        3
0 
id8141  360.242940  149.910199  11950.7
id1594  444.953632  166.985655  11788.4
id1849  364.136849  183.628767  11806.2
id1230  413.836124  184.375703  11916.8
id1948  502.953953  173.237159  12468.3 

# Widths are a list of integers
In [192]: widths = [6, 14, 13, 10]

In [193]: df = pd.read_fwf("bar.csv", widths=widths, header=None)

In [194]: df
Out[194]: 
 0           1           2        3
0  id8141  360.242940  149.910199  11950.7
1  id1594  444.953632  166.985655  11788.4
2  id1849  364.136849  183.628767  11806.2
3  id1230  413.836124  184.375703  11916.8
4  id1948  502.953953  173.237159  12468.3 

In [195]: df = pd.read_fwf("bar.csv", header=None, index_col=0)

In [196]: df
Out[196]: 
 1           2        3
0 
id8141  360.242940  149.910199  11950.7
id1594  444.953632  166.985655  11788.4
id1849  364.136849  183.628767  11806.2
id1230  413.836124  184.375703  11916.8
id1948  502.953953  173.237159  12468.3 

In [197]: pd.read_fwf("bar.csv", header=None, index_col=0).dtypes
Out[197]: 
1    float64
2    float64
3    float64
dtype: object

In [198]: pd.read_fwf("bar.csv", header=None, dtype={2: "object"}).dtypes
Out[198]: 
0     object
1    float64
2     object
3    float64
dtype: object 

In [199]: data = "A,B,C\n20090101,a,1,2\n20090102,b,3,4\n20090103,c,4,5"

In [200]: print(data)
A,B,C
20090101,a,1,2
20090102,b,3,4
20090103,c,4,5

In [201]: with open("foo.csv", "w") as f:
 .....:    f.write(data)
 .....: 

In [202]: pd.read_csv("foo.csv")
Out[202]: 
 A  B  C
20090101  a  1  2
20090102  b  3  4
20090103  c  4  5 

In [203]: df = pd.read_csv("foo.csv", parse_dates=True)

In [204]: df.index
Out[204]: DatetimeIndex(['2009-01-01', '2009-01-02', '2009-01-03'], dtype='datetime64[ns]', freq=None) 

In [205]: data = 'year,indiv,zit,xit\n1977,"A",1.2,.6\n1977,"B",1.5,.5'

In [206]: print(data)
year,indiv,zit,xit
1977,"A",1.2,.6
1977,"B",1.5,.5

In [207]: with open("mindex_ex.csv", mode="w") as f:
 .....:    f.write(data)
 .....: 

In [208]: df = pd.read_csv("mindex_ex.csv", index_col=[0, 1])

In [209]: df
Out[209]: 
 zit  xit
year indiv 
1977 A      1.2  0.6
 B      1.5  0.5

In [210]: df.loc[1977]
Out[210]: 
 zit  xit
indiv 
A      1.2  0.6
B      1.5  0.5 

In [211]: mi_idx = pd.MultiIndex.from_arrays([[1, 2, 3, 4], list("abcd")], names=list("ab"))

In [212]: mi_col = pd.MultiIndex.from_arrays([[1, 2], list("ab")], names=list("cd"))

In [213]: df = pd.DataFrame(np.ones((4, 2)), index=mi_idx, columns=mi_col)

In [214]: df.to_csv("mi.csv")

In [215]: print(open("mi.csv").read())
c,,1,2
d,,a,b
a,b,,
1,a,1.0,1.0
2,b,1.0,1.0
3,c,1.0,1.0
4,d,1.0,1.0

In [216]: pd.read_csv("mi.csv", header=[0, 1, 2, 3], index_col=[0, 1])
Out[216]: 
c                    1                  2
d                    a                  b
a   Unnamed: 2_level_2 Unnamed: 3_level_2
1                  1.0                1.0
2 b                1.0                1.0
3 c                1.0                1.0
4 d                1.0                1.0 

In [217]: data = ",a,a,a,b,c,c\n,q,r,s,t,u,v\none,1,2,3,4,5,6\ntwo,7,8,9,10,11,12"

In [218]: print(data)
,a,a,a,b,c,c
,q,r,s,t,u,v
one,1,2,3,4,5,6
two,7,8,9,10,11,12

In [219]: with open("mi2.csv", "w") as fh:
 .....:    fh.write(data)
 .....: 

In [220]: pd.read_csv("mi2.csv", header=[0, 1], index_col=0)
Out[220]: 
 a         b   c 
 q  r  s   t   u   v
one  1  2  3   4   5   6
two  7  8  9  10  11  12 

In [221]: df = pd.DataFrame(np.random.randn(10, 4))

In [222]: df.to_csv("tmp2.csv", sep=":", index=False)

In [223]: pd.read_csv("tmp2.csv", sep=None, engine="python")
Out[223]: 
 0         1         2         3
0  0.469112 -0.282863 -1.509059 -1.135632
1  1.212112 -0.173215  0.119209 -1.044236
2 -0.861849 -2.104569 -0.494929  1.071804
3  0.721555 -0.706771 -1.039575  0.271860
4 -0.424972  0.567020  0.276232 -1.087401
5 -0.673690  0.113648 -1.478427  0.524988
6  0.404705  0.577046 -1.715002 -1.039268
7 -0.370647 -1.157892 -1.344312  0.844885
8  1.075770 -0.109050  1.643563 -1.469388
9  0.357021 -0.674600 -1.776904 -0.968914 
```### 读取多个文件以创建单个 DataFrame

最好使用`concat()`来合并多个文件。查看 cookbook 以获取示例。  ### 逐块迭代文件

假设您希望惰性地迭代（可能非常大的）文件，而不是将整个文件读入内存，例如以下内容：

```py
In [224]: df = pd.DataFrame(np.random.randn(10, 4))

In [225]: df.to_csv("tmp.csv", index=False)

In [226]: table = pd.read_csv("tmp.csv")

In [227]: table
Out[227]: 
 0         1         2         3
0 -1.294524  0.413738  0.276662 -0.472035
1 -0.013960 -0.362543 -0.006154 -0.923061
2  0.895717  0.805244 -1.206412  2.565646
3  1.431256  1.340309 -1.170299 -0.226169
4  0.410835  0.813850  0.132003 -0.827317
5 -0.076467 -1.187678  1.130127 -1.436737
6 -1.413681  1.607920  1.024180  0.569605
7  0.875906 -2.211372  0.974466 -2.006747
8 -0.410001 -0.078638  0.545952 -1.219217
9 -1.226825  0.769804 -1.281247 -0.727707 

In [228]: with pd.read_csv("tmp.csv", chunksize=4) as reader:
 .....:    print(reader)
 .....:    for chunk in reader:
 .....:        print(chunk)
 .....: 
<pandas.io.parsers.readers.TextFileReader object at 0x7ff2e5421db0>
 0         1         2         3
0 -1.294524  0.413738  0.276662 -0.472035
1 -0.013960 -0.362543 -0.006154 -0.923061
2  0.895717  0.805244 -1.206412  2.565646
3  1.431256  1.340309 -1.170299 -0.226169
 0         1         2         3
4  0.410835  0.813850  0.132003 -0.827317
5 -0.076467 -1.187678  1.130127 -1.436737
6 -1.413681  1.607920  1.024180  0.569605
7  0.875906 -2.211372  0.974466 -2.006747
 0         1         2         3
8 -0.410001 -0.078638  0.545952 -1.219217
9 -1.226825  0.769804 -1.281247 -0.727707 

In [229]: with pd.read_csv("tmp.csv", iterator=True) as reader:
 .....:    print(reader.get_chunk(5))
 .....: 
 0         1         2         3
0 -1.294524  0.413738  0.276662 -0.472035
1 -0.013960 -0.362543 -0.006154 -0.923061
2  0.895717  0.805244 -1.206412  2.565646
3  1.431256  1.340309 -1.170299 -0.226169
4  0.410835  0.813850  0.132003 -0.827317 

df = pd.read_csv("https://download.bls.gov/pub/time.series/cu/cu.item", sep="\t") 

headers = {"User-Agent": "pandas"}
df = pd.read_csv(
    "https://download.bls.gov/pub/time.series/cu/cu.item",
    sep="\t",
    storage_options=headers
) 

df = pd.read_json("s3://pandas-test/adatafile.json") 

storage_options = {"client_kwargs": {"endpoint_url": "http://127.0.0.1:5555"}}}
df = pd.read_json("s3://pandas-test/test-1", storage_options=storage_options) 

pd.read_csv(
    "s3://ncei-wcsd-archive/data/processed/SH1305/18kHz/SaKe2013"
    "-D20130523-T080854_to_SaKe2013-D20130523-T085643.csv",
    storage_options={"anon": True},
) 

pd.read_csv(
    "simplecache::s3://ncei-wcsd-archive/data/processed/SH1305/18kHz/"
    "SaKe2013-D20130523-T080854_to_SaKe2013-D20130523-T085643.csv",
    storage_options={"s3": {"anon": True}},
)