深度 | 强化学习应用金融投资组合优化（附代码）

%load_ext autoreload
%autoreload 2
cd ../acfl/

from env.Env import Env
from env.priceGenerator import make_stock
from learners.a2c import PolicyEstimator,ValueEstimator, reinforce
import tensorflow as tf

def sin_func(num_stocks,length):
    a = np.array(range(length+1))
    a.shape= [length+1,1]
    a = a/(2*np.pi)
    a = np.sin(a)
    return a+5.0001

D = pd.DataFrame(make_stock(1000,10)*np.random.uniform(2,4,10))
plt.xkcd()
D.plot(figsize=(20,10),title='My make beleive market for 10 stocks')

import numpy as np
def make_stock(length=100, num_stocks=2):
    alpha = 0.9
    k = 2
    cov = np.random.normal(0, 5, [num_stocks, num_stocks])
    cov = cov.dot(cov.T) # This is a positive semidefinite matrix, e.g. a covariance matrix
    A = np.random.multivariate_normal(np.zeros(num_stocks), cov, size=[length]) # sample noise, with covariance 
    B = np.random.multivariate_normal(np.zeros(num_stocks), cov, size=[length]) # sample another noise, with covariance
    bs = [np.zeros(shape=num_stocks)] # 
    ps = [np.zeros(shape=num_stocks)] # The prices

    for a, b in zip(A, B):
        bv = alpha * bs[-1] + b # calculate some trend
        bs.append(bv)
        pv = ps[-1] + bs[-2] + k * a # Previosu price + previous trend factor, plus some noise
        ps.append(pv)

    #     ps = [0]
    #     for a,b,common in zip(A,BB,commonNoise):
    #         ps.append(ps[-1]+b+k*a+2*common)
    #     P = np.array(ps)
    #     P = np.exp(P/(P.max()-P.min()))
    ps = np.array(ps).T # reshape it so that its [length,stocks] 
    R = ps.max(1) - ps.min(1) # Scale factor
    prices = np.exp(ps.T / (R)) *np.random.uniform(10,250,num_stocks) # Normalize, exponantiate then make the prices more varied
    return prices

import numpy as np
from collections import  defaultdict

from env.priceGenerator import make_stock

costPerShare = 0 # 0.01
class Env:
    '''
    A simple environemnt for our agent,
    the action our agent gives is  weighting over the stocks + cash
    the env calcutes that into stock and figures out the returns
    '''
    def __init__(self,price_fn,num_stocks=2,length=2,starting_value=1000,lookback=10):
        '''
        :param price_fn:  A function that returns a numpy array of prices
        :param num_stocks: How many stocks in our univerese
        :param length: The length of an episode
        '''
        self.num_stocks = num_stocks
        self.lookback = lookback
        self.length = length
        self.oprices= price_fn(num_stocks=num_stocks,length=length)
        self.prices = np.concatenate([self.oprices,np.ones([length+1,1])],axis=1) #attach the value of cash
        self.portfolio = np.zeros([num_stocks+1]) #2k and 2k+1 are te long and short of a stock. portfolio[-1] is cash
        self.portfolio[-1] = 1
        self.time =0
        self.__account_value = starting_value
        self.__shares=np.array([0]*num_stocks +[starting_value])
        self.hist = defaultdict(list)
    @property
    def shares(self):
        return self.__shares
    @property
    def account_value(self):
        return self.__account_value
    @shares.setter
    def shares(self,new_shares):

        self.__shares = new_shares
        self.hist['shares'].append(self.shares)
    @account_value.setter
    def account_value(self,new_act_val):
        self.__account_value = new_act_val
        try:
            act_returns  = self.account_value / self.hist['act_val'][-1]
        except:
            act_returns =1
        self.hist['act_val'].append(self.account_value)
        self.hist['act_returns'].append(act_returns)

    def step(self,new_portfolio):
        '''
        Get the next prices. Then transition the value of the account into the desired portfolio
        :param new_portfolio:
        :return:
        '''
        self.time +=1

        self.update_acount_value(new_portfolio)
        reward = np.log(self.hist['act_returns'][-1]) #already includes transaction costs
        state = {
            "prices":self.prices[self.time-self.lookback+1:self.time+1,:-1], # All prices upto now inclusive but no cash
            "portfolio":self.portfolio,

        }
        done = self.time >=len(self.prices)-2
        return state,reward,done

    def update_acount_value(self,new_portfolio):
        currentShareValues = self.shares * self.prices[self.time]
        currentAccountValue = sum(currentShareValues)

        currentPortfolioProportions = currentShareValues / currentAccountValue
        desiredCashChange = (new_portfolio -currentPortfolioProportions )* currentAccountValue

        desiredChangeInShares = np.floor(desiredCashChange / self.prices[self.time])
        self.shares = self.shares + desiredChangeInShares
        newAccountValue = np.sum(self.shares*self.prices[self.time])
        #becuse we take the floor, sometimes we lose cash for no reason. This is a fix
        missingCash  = currentAccountValue - newAccountValue
        transactionCost = sum(np.abs(desiredChangeInShares[:-1])*costPerShare)

        self.shares[-1] += missingCash - transactionCost

        transactionCost = sum(np.abs(desiredChangeInShares[:-1])*costPerShare)
        self.hist["changeInShares"].append(desiredChangeInShares)
        self.hist["transactionCosts"].append(transactionCost)

        self.account_value =np.sum(self.shares*self.prices[self.time])

self.alphas= tf.contrib.layers.fully_connected(
          inputs=l2,
          num_outputs=num_stocks+1,
          activation_fn=tf.nn.relu,
          weights_initializer=tf.initializers.glorot_uniform)

      self.alphas +=1

      self.dirichlet  = tfp.distributions.Dirichlet(self.alphas)
      self.action = self.dirichlet._sample_n(1)
      self.action = tf.squeeze(self.action)