Designed manual Seaquest policy network
Public
Designed manual Seaquest policy network
Agent
Codex
Model
GPT 6 Astra
Started
Nov 3, 3:09 AM
Messages3
Downloads0
Outlined policy strategy and hidden unit roles.
PlanDefined network weights and biases manually.
Build################################################################################################################# Authors: ## Kenny Young ([REDACTED]) ## Tian Tian ([REDACTED]) #################################################################################################################import numpy as np
###################################################################################################################### Constants######################################################################################################################ramp_interval = 100max_oxygen = 200init_spawn_speed = 20diver_spawn_speed = 30init_move_interval = 5shot_cool_down = 5enemy_shot_interval = 10enemy_move_interval = 5diver_move_interval = 5
###################################################################################################################### Env## The player controls a submarine consisting of two cells, front and back, to allow direction to be determined. The# player can also fire bullets from the front of the submarine. Enemies consist of submarines and fish, distinguished# by the fact that submarines shoot bullets and fish do not. A reward of +1 is given each time an enemy is struck by# one of the player's bullets, at which point the enemy is also removed. There are also divers which the player can# move onto to pick up, doing so increments a bar indicated by another channel along the bottom of the screen. The# player also has a limited supply of oxygen indicated by another bar in another channel. Oxygen degrades over time,# and is replenished whenever the player moves to the top of the screen as long as the player has at least one rescued# diver on board. The player can carry a maximum of 6 divers. When surfacing with less than 6, one diver is removed.# When surfacing with 6, all divers are removed and a reward is given for each active cell in the oxygen bar. Each# time the player surfaces the difficulty is increased by increasing the spawn rate and movement speed of enemies.# Termination occurs when the player is hit by an enemy fish, sub or bullet; or when oxygen reached 0; or when the# player attempts to surface with no rescued divers. Enemy and diver directions are indicated by a trail channel# active in their previous location to reduce partial observability.######################################################################################################################class Env: def __init__(self, ramping=True): self.channels ={ 'sub_front':0, 'sub_back':1, 'friendly_bullet':2, 'trail':3, 'enemy_bullet':4, 'enemy_fish':5, 'enemy_sub':6, 'oxygen_guage':7, 'diver_guage':8, 'diver':9 } self.action_map = ['n','l','u','r','d','f'] self.ramping = ramping self.random = np.random.RandomState() self.reset()
# Update environment according to agent action def act(self, a): r = 0 if(self.terminal): return r, self.terminal
a = self.action_map[a]
# Spawn enemy if timer is up if(self.e_spawn_timer==0): self._spawn_enemy() self.e_spawn_timer = self.e_spawn_speed
if(self.d_spawn_timer==0): self._spawn_diver() self.d_spawn_timer = diver_spawn_speed
# Resolve player action if(a=='f' and self.shot_timer == 0): self.f_bullets+=[[self.sub_x, self.sub_y, self.sub_or]] self.shot_timer = shot_cool_down elif(a=='l'): self.sub_x = max(0, self.sub_x-1) self.sub_or = False elif(a=='r'): self.sub_x = min(9, self.sub_x+1) self.sub_or = True elif(a=='u'): self.sub_y = max(0, self.sub_y-1) elif(a=='d'): self.sub_y = min(8, self.sub_y+1)
# Update friendly Bullets for bullet in reversed(self.f_bullets): bullet[0]+=1 if bullet[2] else -1 if(bullet[0]<0 or bullet[0]>9): self.f_bullets.remove(bullet) else: removed = False for x in self.e_fish: if(bullet[0:2]==x[0:2]): self.e_fish.remove(x) self.f_bullets.remove(bullet) r+=1 removed = True break if(not removed): for x in self.e_subs: if(bullet[0:2]==x[0:2]): self.e_subs.remove(x) self.f_bullets.remove(bullet) r+=1 break
# Update divers for diver in reversed(self.divers): if(diver[0:2]==[self.sub_x,self.sub_y] and self.diver_count<6): self.divers.remove(diver) self.diver_count+=1 else: if(diver[3]==0): diver[3]=diver_move_interval diver[0]+=1 if diver[2] else -1 if(diver[0]<0 or diver[0]>9): self.divers.remove(diver) elif(diver[0:2]==[self.sub_x,self.sub_y] and self.diver_count<6): self.divers.remove(diver) self.diver_count+=1 else: diver[3]-=1
# Update enemy subs for sub in reversed(self.e_subs): if(sub[0:2]==[self.sub_x,self.sub_y]): self.terminal = True if(sub[3]==0): sub[3]=self.move_speed sub[0]+=1 if sub[2] else -1 if(sub[0]<0 or sub[0]>9): self.e_subs.remove(sub) elif(sub[0:2]==[self.sub_x,self.sub_y]): self.terminal = True else: for x in self.f_bullets: if(sub[0:2]==x[0:2]): self.e_subs.remove(sub) self.f_bullets.remove(x) r+=1 break else: sub[3]-=1 if(sub[4]==0): sub[4]=enemy_shot_interval self.e_bullets+=[[sub[0] if sub[2] else sub[0], sub[1], sub[2]]] else: sub[4]-=1
# Update enemy bullets for bullet in reversed(self.e_bullets): if(bullet[0:2]==[self.sub_x,self.sub_y]): self.terminal = True bullet[0]+=1 if bullet[2] else -1 if(bullet[0]<0 or bullet[0]>9): self.e_bullets.remove(bullet) else: if(bullet[0:2]==[self.sub_x,self.sub_y]): self.terminal = True
# Update enemy fish for fish in reversed(self.e_fish): if(fish[0:2]==[self.sub_x,self.sub_y]): self.terminal = True if(fish[3]==0): fish[3]=self.move_speed fish[0]+=1 if fish[2] else -1 if(fish[0]<0 or fish[0]>9): self.e_fish.remove(fish) elif(fish[0:2]==[self.sub_x,self.sub_y]): self.terminal = True else: for x in self.f_bullets: if(fish[0:2]==x[0:2]): self.e_fish.remove(fish) self.f_bullets.remove(x) r+=1 break else: fish[3]-=1
# Update various timers self.e_spawn_timer -= self.e_spawn_timer>0 self.d_spawn_timer -= self.d_spawn_timer>0 self.shot_timer -= self.shot_timer>0 if(self.oxygen<=0): self.terminal = True if(self.sub_y>0): self.oxygen-=1 self.surface = False else: if(not self.surface): if(self.diver_count == 0): self.terminal = True else: r+=self._surface() return r, self.terminal
# Called when player hits surface (top row) if they have no divers, this ends the game, # if they have 6 divers this gives reward proportional to the remaining oxygen and restores full oxygen # otherwise this reduces the number of divers and restores full oxygen def _surface(self): self.surface = True if(self.diver_count == 6): self.diver_count = 0 r = self.oxygen*10//max_oxygen else: r = 0 self.oxygen = max_oxygen self.diver_count -= 1 if self.ramping and (self.e_spawn_speed>1 or self.move_speed>2): if(self.move_speed>2 and self.ramp_index%2): self.move_speed-=1 if(self.e_spawn_speed>1): self.e_spawn_speed-=1 self.ramp_index+=1 return r
# Spawn an enemy fish or submarine in random row and random direction, # if the resulting row and direction would lead to a collision, do nothing instead def _spawn_enemy(self): lr = self.random.rand() < 1/2 is_sub = self.random.rand() < 1/3 x = 0 if lr else 9 y = self.random.randint(low=1, high=9)
# Do not spawn in same row an opposite direction as existing if(any([z[1]==y and z[2]!=lr for z in self.e_subs+self.e_fish])): return if(is_sub): self.e_subs+=[[x,y,lr,self.move_speed,enemy_shot_interval]] else: self.e_fish+=[[x,y,lr,self.move_speed]]
# Spawn a diver in random row with random direction def _spawn_diver(self): lr = self.random.rand() < 1/2 x = 0 if lr else 9 y = self.random.randint(low=1, high=9) self.divers+=[[x,y,lr,diver_move_interval]]
# Query the current level of the difficulty ramp, could be used as additional input to agent for example def difficulty_ramp(self): return self.ramp_index
# Process the game-state into the 10x10xn state provided to the agent and return def state(self): state = np.zeros((10,10,len(self.channels)),dtype=bool) state[self.sub_y,self.sub_x,self.channels['sub_front']] = 1 back_x = self.sub_x-1 if self.sub_or else self.sub_x+1 state[self.sub_y,back_x,self.channels['sub_back']] = 1 state[9,0:max(0,self.oxygen)*10//max_oxygen, self.channels['oxygen_guage']] = 1 state[9,9-self.diver_count:9, self.channels['diver_guage']] = 1 for bullet in self.f_bullets: state[bullet[1],bullet[0], self.channels['friendly_bullet']] = 1 for bullet in self.e_bullets: state[bullet[1],bullet[0], self.channels['enemy_bullet']] = 1 for fish in self.e_fish: state[fish[1],fish[0], self.channels['enemy_fish']] = 1 back_x = fish[0]-1 if fish[2] else fish[0]+1 if(back_x>=0 and back_x<=9): state[fish[1],back_x, self.channels['trail']] = 1 for sub in self.e_subs: state[sub[1],sub[0], self.channels['enemy_sub']] = 1 back_x = sub[0]-1 if sub[2] else sub[0]+1 if(back_x>=0 and back_x<=9): state[sub[1],back_x, self.channels['trail']] = 1 for diver in self.divers: state[diver[1],diver[0], self.channels['diver']] = 1 back_x = diver[0]-1 if diver[2] else diver[0]+1 if(back_x>=0 and back_x<=9): state[diver[1],back_x, self.channels['trail']] = 1
return state
# Reset to start state for new episode def reset(self): self.oxygen = max_oxygen self.diver_count = 0 self.sub_x = 5 self.sub_y = 0 # 0=left, 1=right self.sub_or = False self.f_bullets = [] self.e_bullets = [] self.e_fish = [] self.e_subs = [] self.divers = [] self.e_spawn_speed = init_spawn_speed self.e_spawn_timer = self.e_spawn_speed self.d_spawn_timer = diver_spawn_speed self.move_speed = init_move_interval self.ramp_index = 0 self.shot_timer = 0 self.surface = True self.terminal = False
# Dimensionality of the game-state (10x10xn) def state_shape(self): return [10,10,len(self.channels)]
# Subset of actions that actually have a unique impact in this environment def minimal_action_set(self): minimal_actions = ['n','l','u','r','d','f'] return [self.action_map.index(x) for x in minimal_actions]W1[12, 0] = 2.5 # one entry: row 12, column 0W1[0:6, 3] = [1, 0, -1, 0, 2, 0] # explicit values along a range (as many values as entries)b1[0:4] = -1 # fill a range (a:b is half-open; ':' alone = the whole axis)W1[3:600:60, 0] = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1] # column 0 of channel 3, rows 0..9 of the grid -> unit 0W2[0:2, 0:3] = [[1, -1, 0.5], [0, 2, 0]] # a 2-D list for a 2-D block (here 2 x 3)W3[:, 2] *= 2 # scale (also += and -=)# First layer: player row indicators.b1[0:9] = -6W1[0:60:6, 0] = 6W1[60:120:6, 1] = 6W1[120:180:6, 2] = 6W1[180:240:6, 3] = 6W1[240:300:6, 4] = 6W1[300:360:6, 5] = 6W1[360:420:6, 6] = 6W1[420:480:6, 7] = 6W1[480:540:6, 8] = 6
# Diver presence in rows 2 through 8.b1[9:16] = -4W1[122:180:6, 9] = 8W1[121:180:6, 9] = -8W1[182:240:6, 10] = 8W1[181:240:6, 10] = -8W1[242:300:6, 11] = 8W1[241:300:6, 11] = -8W1[302:360:6, 12] = 8W1[301:360:6, 12] = -8W1[362:420:6, 13] = 8W1[361:420:6, 13] = -8W1[422:480:6, 14] = 8W1[421:480:6, 14] = -8W1[482:540:6, 15] = 8W1[481:540:6, 15] = -8
# Collision risk near the fixed player column.b1[16:32] = -3W1[61:120:6, 16] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[62:120:6, 16] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[121:180:6, 17] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[122:180:6, 17] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[181:240:6, 18] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[182:240:6, 18] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[241:300:6, 19] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[242:300:6, 19] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[301:360:6, 20] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[302:360:6, 20] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[361:420:6, 21] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[362:420:6, 21] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[421:480:6, 22] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[422:480:6, 22] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]W1[481:540:6, 23] = [0, 0, 2, 4, 7, 7, 7, 5, 2, 0]W1[482:540:6, 23] = [0, 0, -1.8, -3.6, -1, 0, -1, -3, -1.8, 0]
# Surface request: oxygen below 60 and at least one diver.W1[557, 24] = -6W1[593, 24] = 6
# Left-side shooting opportunities in rows 2 through 8.W1[121:144:6, 25] = 6W1[181:204:6, 26] = 6W1[241:264:6, 27] = 6W1[301:324:6, 28] = 6W1[361:384:6, 29] = 6W1[421:444:6, 30] = 6W1[481:504:6, 31] = 6
# Second layer: fire candidate gated by current row.W2[1, 0] = 10W2[2, 1] = 10W2[3, 2] = 10W2[4, 3] = 10W2[5, 4] = 10W2[6, 5] = 10W2[7, 6] = 10W2[8, 7] = 10
# Up candidates.W2[2, 8] = 10W2[3, 9] = 10W2[4, 10] = 10W2[5, 11] = 10W2[6, 12] = 10W2[7, 13] = 10W2[8, 14] = 10W2[1, 15] = 6
# Down candidates.W2[0, 16] = 10W2[1, 17] = 10W2[2, 18] = 10W2[3, 19] = 10W2[4, 20] = 10W2[5, 21] = 10W2[6, 22] = 10W2[7, 23] = 10
# Diver attraction for fire candidates.W2[9:16, 0:8] = [[0.45, 0.6, 0.45, 0.34, 0.255, 0.19, 0.145, 0.11],[0.34, 0.45, 0.6, 0.45, 0.34, 0.255, 0.19, 0.145],[0.255, 0.34, 0.45, 0.6, 0.45, 0.34, 0.255, 0.19],[0.19, 0.255, 0.34, 0.45, 0.6, 0.45, 0.34, 0.255],[0.145, 0.19, 0.255, 0.34, 0.45, 0.6, 0.45, 0.34],[0.11, 0.145, 0.19, 0.255, 0.34, 0.45, 0.6, 0.45],[0.08, 0.11, 0.145, 0.19, 0.255, 0.34, 0.45, 0.6]]
# Diver attraction for up candidates.W2[9:16, 8:15] = [[0.45, 0.6, 0.45, 0.34, 0.255, 0.19, 0.145],[0.34, 0.45, 0.6, 0.45, 0.34, 0.255, 0.19],[0.255, 0.34, 0.45, 0.6, 0.45, 0.34, 0.255],[0.19, 0.255, 0.34, 0.45, 0.6, 0.45, 0.34],[0.145, 0.19, 0.255, 0.34, 0.45, 0.6, 0.45],[0.11, 0.145, 0.19, 0.255, 0.34, 0.45, 0.6],[0.08, 0.11, 0.145, 0.19, 0.255, 0.34, 0.45]]
# Diver attraction for down candidates.W2[9:16, 16:24] = [[0.45, 0.6, 0.45, 0.34, 0.255, 0.19, 0.145, 0.11],[0.34, 0.45, 0.6, 0.45, 0.34, 0.255, 0.19, 0.145],[0.255, 0.34, 0.45, 0.6, 0.45, 0.34, 0.255, 0.19],[0.19, 0.255, 0.34, 0.45, 0.6, 0.45, 0.34, 0.255],[0.145, 0.19, 0.255, 0.34, 0.45, 0.6, 0.45, 0.34],[0.11, 0.145, 0.19, 0.255, 0.34, 0.45, 0.6, 0.45],[0.08, 0.11, 0.145, 0.19, 0.255, 0.34, 0.45, 0.6]]
# Shooting opportunity attraction for fire candidates.W2[25:32, 0:8] = [[0.12, 0.35, 0.12, 0.04, 0.01, 0, 0, 0],[0.04, 0.12, 0.35, 0.12, 0.04, 0.01, 0, 0],[0.01, 0.04, 0.12, 0.35, 0.12, 0.04, 0.01, 0],[0, 0.01, 0.04, 0.12, 0.35, 0.12, 0.04, 0.01],[0, 0, 0.01, 0.04, 0.12, 0.35, 0.12, 0.04],[0, 0, 0, 0.01, 0.04, 0.12, 0.35, 0.12],[0, 0, 0, 0, 0.01, 0.04, 0.12, 0.35]]
# Shooting opportunity attraction for up candidates.W2[25:32, 8:15] = [[0.12, 0.35, 0.12, 0.04, 0.01, 0, 0],[0.04, 0.12, 0.35, 0.12, 0.04, 0.01, 0],[0.01, 0.04, 0.12, 0.35, 0.12, 0.04, 0.01],[0, 0.01, 0.04, 0.12, 0.35, 0.12, 0.04],[0, 0, 0.01, 0.04, 0.12, 0.35, 0.12],[0, 0, 0, 0.01, 0.04, 0.12, 0.35],[0, 0, 0, 0, 0.01, 0.04, 0.12]]
# Shooting opportunity attraction for down candidates.W2[25:32, 16:24] = [[0.12, 0.35, 0.12, 0.04, 0.01, 0, 0, 0],[0.04, 0.12, 0.35, 0.12, 0.04, 0.01, 0, 0],[0.01, 0.04, 0.12, 0.35, 0.12, 0.04, 0.01, 0],[0, 0.01, 0.04, 0.12, 0.35, 0.12, 0.04, 0.01],[0, 0, 0.01, 0.04, 0.12, 0.35, 0.12, 0.04],[0, 0, 0, 0.01, 0.04, 0.12, 0.35, 0.12],[0, 0, 0, 0, 0.01, 0.04, 0.12, 0.35]]
# Immediate destination hazards.W2[16, 0:24:8] = -3W2[17, 1:24:8] = -3W2[18, 2:24:8] = -3W2[19, 3:24:8] = -3W2[20, 4:24:8] = -3W2[21, 5:24:8] = -3W2[22, 6:24:8] = -3W2[23, 7] = -3W2[23, 23] = -3
# Additional risk if a vertical action sticks.W2[16, 9] = -0.3W2[17, 10] = -0.3W2[18, 11] = -0.3W2[19, 12] = -0.3W2[20, 13] = -0.3W2[21, 14] = -0.3W2[17, 16] = -0.3W2[18, 17] = -0.3W2[19, 18] = -0.3W2[20, 19] = -0.3W2[21, 20] = -0.3W2[22, 21] = -0.3W2[23, 22] = -0.3
# Oxygen-driven ascent, gated against empty surfacing.W2[24, 8] = 4.75W2[24, 9:15] = 0.75W2[24, 15] = 6
# Extra surface guard near rows 1 and 2.W2[1:3, 24] = 3W2[24, 24] = -3
# Leave row 1 promptly unless ascending for oxygen.W2[1, 25] = 3W2[24, 25] = -3
# Candidate offsets.b2[0:8] = [-11.96, -11.09, -10.63, -10.395, -10.32, -10.395, -10.63, -11.09]b2[8:16] = [-15.21, -10.64, -10.18, -9.945, -9.87, -9.945, -10.18, -6]b2[16:24] = [-12.26, -11.39, -10.93, -10.695, -10.62, -10.695, -10.93, -11.09]b2[24] = 0b2[25] = -3
# Action logits.W3[0:8, 5] = 30W3[8:15, 2] = 30W3[15, 2] = 40W3[16:24, 4] = 30W3[24, 2] = -12W3[25, 4] = 4b3[:] = [-100, -100, 238, -100, 244, 241.8]