[research/lotterysim] sigmoid strategy, and miscel changes

script/research/lotterysim/RPID.py

@@ -1,10 +1,10 @@
 from utils import *
 
 REWARD_MIN = 0
-REWARD_MAX = 1000
+REWARD_MAX = 10000
 
 class RPID:
-    def __init__(self, kp=0, ki=0, kd=0, dt=1, target=15, Kc=0, Ti=0, Td=0, Ts=0, debug=False):
+    def __init__(self, kp=0, ki=0, kd=0, dt=1, target_apy=10, Kc=0, Ti=0, Td=0, Ts=0, debug=False):
         self.Kp = kp # discrete pid kp
         self.Ki = ki # discrete pid ki
         self.Kd = kd # discrete pid kd
@@ -13,7 +13,7 @@ class RPID:
         self.Td = Td # takahashi td
         self.Ts = Ts # takahashi ts
         self.Kc = Kc # takahashi kc
-        self.target = target # pid set point
+        self.target = target_apy # pid set point, target APY
         self.prev_feedback = 0
         self.feedback_hist = [0, 0]
         self.f_hist = [0]
@@ -46,16 +46,16 @@ class RPID:
         else:
             pid_value = self.pid(feedback)
 
-        if pid_value <= 0.0:
-            pid_value = REWARD_MIN
-        elif pid_value >= 1:
-            pid_value =  REWARD_MAX
+        if pid_value <= REWARD_MIN:
+            pid_value = REWARD_MIN # should be corresponding to lowest APY
+        if pid_value >= REWARD_MAX:
+            pid_value =  REWARD_MAX # should be corresponding to target APY
 
         self.f_hist+=[pid_value]
         return pid_value
 
     def error(self, feedback):
-        return feedback - self.target
+        return self.target - feedback
 
     def proportional(self,  feedback):
         return self.error(feedback)

script/research/lotterysim/auto_crawler.py

@@ -2,9 +2,9 @@ from lottery import *
 from threading import Thread
 from argparse import ArgumentParser
 
-AVG_LEN = 5
+avg_len = 5
 
-KP_STEP=0.01
+kp_STEP=0.01
 KP_SEARCH= -0.04019999999996926
 
 KI_STEP=0.01
@@ -47,7 +47,6 @@ def experiment(accs=[], controller_type=CONTROLLER_TYPE_DISCRETE, kp=0, ki=0, kd
         darkie = Darkie(distribution[idx])
         dt.add_darkie(darkie)
     acc = dt.background(rand_running_time, hp)
-    accs+=[acc]
     return acc
 
 def multi_trial_exp(kp, ki, kd, distribution = [], hp=True):

script/research/lotterysim/auto_crawler_takahashi.py

@@ -52,7 +52,6 @@ def experiment(accs=[], controller_type=CONTROLLER_TYPE_TAKAHASHI, kp=0, ki=0, k
         darkie = Darkie(distribution[idx])
         dt.add_darkie(darkie)
     acc = dt.background(rand_running_time, hp)
-    accs+=[acc]
     return acc
 
 

script/research/lotterysim/constants.py

@@ -7,12 +7,11 @@ CONTROLLER_TYPE_TAKAHASHI=1
 ERC20DRK=2.1*10**9
 
 L = 28948022309329048855892746252171976963363056481941560715954676764349967630337.0
-REWARD = 1000
+REWARD = 1
 F_MIN = 0.0001
 F_MAX = 0.9999
 EPSILON = 1
 EPOCH_LENGTH = 10
-
 L_HP = Num(L)
 REWARD_HP = Num(REWARD)
 F_MIN_HP = Num(F_MIN)

script/research/lotterysim/darkie.py

@@ -3,7 +3,7 @@ from threading import Thread
 from strategy import *
 
 class Darkie(Thread):
-    def __init__(self, airdrop, initial_stake=None, vesting=[], hp=False, commit=True, epoch_len=100, strategy=None, apy_window=0):
+    def __init__(self, airdrop, initial_stake=None, vesting=[], hp=False, commit=True, epoch_len=100, strategy=None, apy_window=EPOCH_LENGTH):
         Thread.__init__(self)
         self.vesting = [0] + vesting
         self.stake = (Num(airdrop) if hp else airdrop)
@@ -23,14 +23,19 @@ class Darkie(Thread):
         return Darkie(self.finalized_stake)
 
     def apy(self):
+        window = 0
         if self.apy_window == 0:
             window=len(self.initial_stake)
         # approximation to APY assuming linear relation
         # note! relation is logarithmic depending on PID output.
         if window<len(self.initial_stake):
             windowed_initial_stake = self.initial_stake[-window]
-            return Num(self.stake - windowed_initial_stake) / Num(windowed_initial_stake) if self.stake>0 else Num(0)
-        return Num(self.stake - self.initial_stake[0]) / Num(self.initial_stake[0]) if self.stake>0 else Num(0)
+            apy =  Num(self.stake - windowed_initial_stake) / Num(windowed_initial_stake) if self.stake>0 else Num(0)
+            #print('slot: {}, windowed apy: {}, gain: {}'.format(self.slot, apy, self.stake-windowed_initial_stake))
+            return apy
+        apy = Num(self.stake - self.initial_stake[0]) / Num(self.initial_stake[0]) if self.stake>0 else Num(0)
+        print('slot: {}, apy: {}, gain: {}'.format(self.slot, apy, self.stake-self.initial_stake[0]))
+        return apy
 
     def apy_percentage(self):
         return self.apy()*100
@@ -51,7 +56,7 @@ class Darkie(Thread):
             scaled_target = approx_target_in_zk(sigmas, Num(stake)) #+ (BASE_L_HP if hp else BASE_L)
             return scaled_target
 
-        self.strategy.set_ratio(self.slot, self.apy())
+        self.strategy.set_ratio(self.slot, self.apy_percentage())
         T = target(self.f, self.strategy.staked_value(self.finalized_stake))
         self.won = lottery(T, hp)
 

script/research/lotterysim/highest_gain.txt

@@ -1 +1 @@
-accuracy:0.5613850265422056, kp: 1.5000000000000024, ki:1, kd:-1
+avg(acc): 0.12014555104338387, avg(apy): 0.3250504347682971, avg(reward): 576.0759327273176, avg(stake ratio): 0.20001903883969724, kp: 0.13499999999919243, ki:5.180999999999765, kd:-0.11000000000000018

script/research/lotterysim/instance.py

@@ -7,19 +7,20 @@ os.system("rm f.hist; rm leads.hist")
 
 RUNNING_TIME = int(input("running time:"))
 
-NODES=1000
+NODES=10
 
 if __name__ == "__main__":
     darkies = []
     egalitarian = ERC20DRK/NODES
-    darkies += [ Darkie(random.gauss(egalitarian, egalitarian*0.1), strategy=LinearStrategy(EPOCH_LENGTH)) for id in range(int(NODES)) ]
-    darkies += [Darkie(0, strategy=LinearStrategy(EPOCH_LENGTH)) for _ in range(NODES)]
+    darkies += [ Darkie(random.gauss(egalitarian, egalitarian*0.1), strategy=SigmoidStrategy(EPOCH_LENGTH), apy_window=EPOCH_LENGTH) for id in range(int(NODES)) ]
+    #TODO try rpid with 0mint
+    #darkies += [Darkie(0, strategy=LinearStrategy(EPOCH_LENGTH)) for _ in range(NODES)]
     airdrop = ERC20DRK
     effective_airdrop  = 0
     for darkie in darkies:
         effective_airdrop+=darkie.stake
     print("network airdrop: {}, staked token: {}/{}% on {} nodes".format(airdrop, effective_airdrop, effective_airdrop/airdrop*100, len(darkies)))
-    dt = DarkfiTable(airdrop, RUNNING_TIME, CONTROLLER_TYPE_DISCRETE, kp=-0.010399999999938556, ki=-0.0365999996461878, kd=0.03840000000000491)
+    dt = DarkfiTable(airdrop, RUNNING_TIME, CONTROLLER_TYPE_DISCRETE, kp=-0.010399999999938556, ki=-0.0365999996461878, kd=0.03840000000000491,  r_kp=4.935, r_ki=0.429, r_kd=-0.05)
     for darkie in darkies:
         dt.add_darkie(darkie)
     acc = dt.background(rand_running_time=False)

script/research/lotterysim/leads.hist

@@ -1 +1 @@
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
+0,0,0.0,2.0,1.0,1.0,1.0,1.0,0.0,3.0,1.0,2.0,1.0,1.0,2.0,1.0,0.0,0.0,0.0,1.0,2.0,1.0,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,

script/research/lotterysim/lottery.py

@@ -7,7 +7,7 @@ from pid import PID
 from RPID import RPID
 
 class DarkfiTable:
-    def __init__(self, airdrop, running_time, controller_type=CONTROLLER_TYPE_DISCRETE, kp=0, ki=0, kd=0, dt=1, target=1, reward_target=15, kc=0, ti=0, td=0, ts=0, debug=False, r_kp=0, r_ki=0, r_kd=0):
+    def __init__(self, airdrop, running_time, controller_type=CONTROLLER_TYPE_DISCRETE, kp=0, ki=0, kd=0, dt=1, target=1, reward_target=10, kc=0, ti=0, td=0, ts=0, debug=False, r_kp=0, r_ki=0, r_kd=0):
         self.Sigma=airdrop
         self.darkies = []
         self.running_time=running_time
@@ -15,7 +15,7 @@ class DarkfiTable:
         self.end_time=None
         self.pid = None
         self.pid = PID(kp=kp, ki=ki, kd=kd, dt=dt, target=target, Kc=kc, Ti=ti, Td=td, Ts=ts)
-        self.rpid = RPID(kp=r_kp, ki=r_ki, kd=r_kd, target=reward_target)
+        self.rpid = RPID(kp=r_kp, ki=r_ki, kd=r_kd, target_apy=reward_target)
         self.controller_type=controller_type
         self.debug=debug
 
@@ -33,6 +33,7 @@ class DarkfiTable:
         #if rand_running_time and debug:
             #print("random running time: {}".format(self.running_time))
             #print('running time: {}'.format(self.running_time))
+        rewards = [0]
         while count < self.running_time:
             winners=0
             total_vesting_stake = 0
@@ -44,9 +45,13 @@ class DarkfiTable:
                 total_vesting_stake+=self.darkies[i].update_vesting()
             for i in range(len(self.darkies)):
                 winners += self.darkies[i].won
-                apy = self.darkies[i].apy()
-                reward = self.rpid.pid_clipped(float(apy), self.controller_type, debug)
-                self.darkies[i].update_stake(reward)
+                print('reward: {}'.format(rewards[-1]))
+                self.darkies[i].update_stake(rewards[-1])
+
+            if count%EPOCH_LENGTH == 0:
+                acc = self.pid.acc()
+                reward = self.rpid.pid_clipped(float(self.avg_apy()), self.controller_type, debug)
+                rewards += [reward]
             feedback = winners
             if winners==1:
                 if count >= ERC20DRK:
@@ -57,6 +62,55 @@ class DarkfiTable:
         self.end_time=time.time()
         return self.pid.acc()
 
+    def background_with_apy(self, rand_running_time=True, debug=False, hp=True):
+        self.debug=debug
+        self.start_time=time.time()
+        feedback=0 # number leads in previous slot
+        count = 0
+        # random running time
+        rand_running_time = random.randint(1,self.running_time) if rand_running_time else self.running_time
+        self.running_time = rand_running_time
+        #if rand_running_time and debug:
+            #print("random running time: {}".format(self.running_time))
+            #print('running time: {}'.format(self.running_time))
+        rewards = [0]
+        while count < self.running_time:
+            winners=0
+            total_vesting_stake = 0
+            f = self.pid.pid_clipped(float(feedback), self.controller_type, debug)
+
+            #note! thread overhead is 10X slower than sequential node execution!
+            for i in range(len(self.darkies)):
+                self.darkies[i].set_sigma_feedback(self.Sigma, feedback, f, count, hp)
+                self.darkies[i].run(hp)
+                total_vesting_stake+=self.darkies[i].update_vesting()
+
+            #print('reward: {}'.format(rewards[-1]))
+            for i in range(len(self.darkies)):
+                winners += self.darkies[i].won
+                self.darkies[i].update_stake(rewards[-1])
+                ###
+
+            if count%EPOCH_LENGTH == 0 :
+                acc = self.pid.acc()
+                reward = self.rpid.pid_clipped(float(self.avg_apy()), self.controller_type, debug)
+                rewards += [reward]
+
+            feedback = winners
+            if winners==1:
+                if count >= ERC20DRK:
+                    self.Sigma += 1
+                for i in range(len(self.darkies)):
+                    self.darkies[i].finalize_stake()
+            count+=1
+        self.end_time=time.time()
+        avg_reward = sum(rewards)/len(rewards)
+        avg_stake_ratio = sum([darkie.strategy.staked_tokens_ratio for darkie in self.darkies])/len(self.darkies)
+        return self.pid.acc(), self.avg_apy()*Num(ONE_YEAR/self.running_time), avg_reward, avg_stake_ratio
+
+    def avg_apy(self):
+        return sum([darkie.apy_percentage() for darkie in self.darkies])/len(self.darkies)
+
     def write(self):
         elapsed=self.end_time-self.start_time
         if self.debug:

script/research/lotterysim/reward_auto_crawler.py

@@ -4,19 +4,21 @@ from argparse import ArgumentParser
 
 AVG_LEN = 5
 
-KP_STEP=0.5
-KP_SEARCH=1
+KP_STEP=0.3
+KP_SEARCH=4.935 #-0.91
 
-KI_STEP=0.5
-KI_SEARCH=1
+KI_STEP=0.3
+KI_SEARCH=0.429 #157.5
 
-KD_STEP=0.5
-KD_SEARCH=-1
+KD_STEP=0.3
+KD_SEARCH=-0.05 #5.6
 
 EPSILON=0.0001
+
 RUNNING_TIME=1000
-NODES = 1000
+NODES = 100
 
+highest_apy = 0
 highest_acc = 0
 
 KP='kp'
@@ -40,32 +42,42 @@ randomize_nodes = args.randomize_nodes
 rand_running_time = args.rand_running_time
 debug = args.debug
 
-def experiment(accs=[], controller_type=CONTROLLER_TYPE_DISCRETE, kp=0, ki=0, kd=0, distribution=[], hp=True):
-    dt = DarkfiTable(ERC20DRK, RUNNING_TIME, controller_type, kp=-0.010399999999938556, ki=-0.0365999996461878, kd=0.03840000000000491, r_kp=kp, r_ki=ki, r_kd=kd)
+def experiment(apys=[], controller_type=CONTROLLER_TYPE_DISCRETE, rkp=0, rki=0, rkd=0, distribution=[], hp=True):
+    dt = DarkfiTable(ERC20DRK, RUNNING_TIME, controller_type, kp=-0.010399999999938556, ki=-0.0365999996461878, kd=0.03840000000000491, r_kp=rkp, r_ki=rki, r_kd=rkd)
     RND_NODES = random.randint(5, NODES) if randomize_nodes else NODES
     for idx in range(0,RND_NODES):
-        darkie = Darkie(distribution[idx])
+        darkie = Darkie(distribution[idx], strategy=SigmoidStrategy(EPOCH_LENGTH), apy_window=EPOCH_LENGTH)
         dt.add_darkie(darkie)
-    acc = dt.background(rand_running_time, hp)
-    accs+=[acc]
-    return acc
+    acc, apy, reward, stake_ratio = dt.background_with_apy(rand_running_time, hp)
+    return acc, apy, reward, stake_ratio
 
 def multi_trial_exp(kp, ki, kd, distribution = [], hp=True):
+    global highest_apy
     global highest_acc
     global highest_gain
     new_record=False
     exp_threads = []
     accs = []
+    apys = []
+    rewards = []
+    stakes_ratios = []
     for i in range(0, AVG_LEN):
-        acc = experiment(accs, CONTROLLER_TYPE_DISCRETE, kp=kp, ki=ki, kd=kd, distribution=distribution, hp=hp)
+        acc, apy, reward, stake_ratio = experiment(apys, CONTROLLER_TYPE_DISCRETE, rkp=kp, rki=ki, rkd=kd, distribution=distribution, hp=hp)
         accs += [acc]
-    avg_acc = sum(accs)/float(AVG_LEN)
-    buff = 'accuracy:{}, kp: {}, ki:{}, kd:{}'.format(avg_acc, kp, ki, kd)
-    if avg_acc > 0:
+        apys += [apy]
+        rewards += [reward]
+        stakes_ratios += [stake_ratio]
+    avg_acc = float(sum(accs))/len(accs)
+    avg_apy = float(sum(apys))/float(AVG_LEN)
+    avg_reward = float(sum(rewards))/len(rewards)
+    avg_staked = float(sum(stakes_ratios))/len(stakes_ratios)
+    buff = 'avg(acc): {}, avg(apy): {}, avg(reward): {}, avg(stake ratio): {}, kp: {}, ki:{}, kd:{}'.format(avg_acc, avg_apy, avg_reward, avg_staked, kp, ki, kd)
+    if avg_apy > 0:
         gain = (kp, ki, kd)
-        acc_gain = (avg_acc, gain)
-        if avg_acc > highest_acc:
+        acc_gain = (avg_apy, gain)
+        if avg_apy > highest_apy and avg_acc > highest_acc:
             new_record = True
+            highest_apy = avg_apy
             highest_acc = avg_acc
             highest_gain = (kp, ki, kd)
             with open("highest_gain.txt", 'w') as f:
@@ -100,7 +112,7 @@ def crawler(crawl, range_multiplier, step=0.1):
         ki = i if crawl==KI else highest_gain[1]
         kd = i if crawl==KD else highest_gain[2]
         buff, new_record = multi_trial_exp(kp, ki, kd, distribution, hp=high_precision)
-        crawl_range.set_description('highest:{} / {}'.format(highest_acc, buff))
+        crawl_range.set_description('highest:{} / {}'.format(highest_apy, buff))
         if new_record:
             break
 
@@ -116,8 +128,8 @@ while True:
         range_start = (start*KP_RANGE_MULTIPLIER if start <=0 else -1*start) - SHIFTING
         range_end = (-1*start if start<=0 else KP_RANGE_MULTIPLIER*start) + SHIFTING
         while (range_end - range_start)/KP_STEP >500:
-            if KP_STEP < 0.1:
-                KP_STEP*=10
+            #if KP_STEP < 0.1:
+            KP_STEP*=2
             KP_RANGE_MULTIPLIER-=1
             #TODO (res) shouldn't the range also shrink?
             # not always true.
@@ -136,8 +148,9 @@ while True:
         range_start = (start*KI_RANGE_MULTIPLIER if start <=0 else -1*start) - SHIFTING
         range_end = (-1*start if start<=0 else KI_RANGE_MULTIPLIER*start) + SHIFTING
         while (range_end - range_start)/KI_STEP >500:
-            if KP_STEP < 0.1:
-                KI_STEP*=10
+            #print('range_end: {}, range_start: {}, ki_step: {}'.format(range_end, range_start, KI_STEP))
+            #if KP_STEP < 1:
+            KI_STEP*=2
             KI_RANGE_MULTIPLIER-=1
     # kd crawl
     crawler(KD, KD_RANGE_MULTIPLIER, KD_STEP)
@@ -149,6 +162,6 @@ while True:
         range_start = (start*KD_RANGE_MULTIPLIER if start <=0 else -1*start) - SHIFTING
         range_end = (-1*start if start<=0 else KD_RANGE_MULTIPLIER*start) + SHIFTING
         while (range_end - range_start)/KD_STEP >500:
-            if KD_STEP < 0.1:
-                KD_STEP*=10
+            #if KD_STEP < 0.1:
+            KD_STEP*=2
             KD_RANGE_MULTIPLIER-=1

script/research/lotterysim/strategy.py

@@ -1,4 +1,5 @@
 from utils import *
+import math
 
 class Strategy(object):
     def __init__(self, epoch_len=0):
@@ -17,8 +18,9 @@ class RandomStrategy(Strategy):
 
 
     def set_ratio(self, slot, apy=0):
-        if slot%self.epoch_len==0:
+        if slot%self.epoch_len==0 and slot>EPOCH_LENGTH:
             self.staked_tokens_ratio = random.random()
+            #print('staked ratio: {}'.format(self.staked_tokens_ratio))
 
 class LinearStrategy(Strategy):
     '''
@@ -27,8 +29,42 @@ class LinearStrategy(Strategy):
     '''
     def __init__(self, epoch_len=0):
         Strategy.__init__(self, epoch_len)
-        self.TARGET_APY = 20.0
+        self.TARGET_APY = 15
 
     def set_ratio(self, slot, apy):
-        if slot%self.epoch_len==0:
+        if slot%self.epoch_len==0 and slot>EPOCH_LENGTH:
             self.staked_tokens_ratio = apy/Num(self.TARGET_APY)
+            #print('staked ratio: {}'.format(self.staked_tokens_ratio))
+
+class LogarithmicStrategy(Strategy):
+    '''
+    logarithmic staking strategy wrt apy.
+    assume optimal is 20% APY!
+    '''
+    def __init__(self, epoch_len=0):
+        Strategy.__init__(self, epoch_len)
+        self.TARGET_APY = 15
+
+    def set_ratio(self, slot, apy):
+        if slot%self.epoch_len==0 and slot>EPOCH_LENGTH:
+            apy_ratio = math.fabs(apy/self.TARGET_APY)
+            self.staked_tokens_ratio = Num((math.log(apy_ratio, 10)+1)/2 if apy_ratio != 0 else 0)
+            #print('staked ratio: {}'.format(self.staked_tokens_ratio))
+
+class SigmoidStrategy(Strategy):
+    '''
+    logarithmic staking strategy wrt apy.
+    assume optimal is 20% APY!
+    '''
+    def __init__(self, epoch_len=0):
+        Strategy.__init__(self, epoch_len)
+        self.TARGET_APY = 10
+
+    def set_ratio(self, slot, apy):
+        if slot%self.epoch_len==0 and slot>self.epoch_len:
+            apy_ratio = math.fabs(apy/self.TARGET_APY)
+            #print("apy ratio: {}".format(apy_ratio))
+            self.staked_tokens_ratio = Num(2/(1+math.pow(math.e, -4*apy_ratio))-1)
+            #print('ratio: {}, staked: {}'.format(apy_ratio, self.staked_tokens_ratio))
+            assert(self.staked_tokens_ratio>=0 and self.staked_tokens_ratio<=1)
+            #print('staked ratio: {}'.format(self.staked_tokens_ratio))