################################################################################################################# Authors: ## Kenny Young ([REDACTED]) ## Tian Tian ([REDACTED]) #################################################################################################################import numpy as np
###################################################################################################################### Env## The player controls a paddle on the bottom of the screen and must bounce a ball tobreak 3 rows of bricks along the# top of the screen. A reward of +1 is given for each brick broken by the ball. When all bricks are cleared another 3# rows are added. The ball travels only along diagonals, when it hits the paddle it is bounced either to the left or# right depending on the side of the paddle hit, when it hits a wall or brick it is reflected. Termination occurs when# the ball hits the bottom of the screen. The balls direction is indicated by a trail channel.######################################################################################################################class Env: def __init__(self, ramping=None): self.channels ={ 'paddle':0, 'ball':1, 'trail':2, 'brick':3, } self.action_map = ['n','l','u','r','d','f'] self.random = np.random.RandomState() self.reset()
# Update environment according to agent action def act(self, a): r = 0 if(self.terminal): return r, self.terminal
a = self.action_map[a]
# Resolve player action if(a=='l'): self.pos = max(0, self.pos-1) elif(a=='r'): self.pos = min(9,self.pos+1)
# Update ball position self.last_x = self.ball_x self.last_y = self.ball_y if(self.ball_dir == 0): new_x = self.ball_x-1 new_y = self.ball_y-1 elif(self.ball_dir == 1): new_x = self.ball_x+1 new_y = self.ball_y-1 elif(self.ball_dir == 2): new_x = self.ball_x+1 new_y = self.ball_y+1 elif(self.ball_dir == 3): new_x = self.ball_x-1 new_y = self.ball_y+1
strike_toggle = False if(new_x<0 or new_x>9): if(new_x<0): new_x = 0 if(new_x>9): new_x=9 self.ball_dir=[1,0,3,2][self.ball_dir] if(new_y<0): new_y = 0 self.ball_dir=[3,2,1,0][self.ball_dir] elif(self.brick_map[new_y,new_x]==1): strike_toggle = True if(not self.strike): r+=1 self.strike = True self.brick_map[new_y,new_x]=0 new_y = self.last_y self.ball_dir=[3,2,1,0][self.ball_dir] elif(new_y == 9): if(np.count_nonzero(self.brick_map)==0): self.brick_map[1:4,:] = 1 if(self.ball_x == self.pos): self.ball_dir=[3,2,1,0][self.ball_dir] new_y = self.last_y elif(new_x == self.pos): self.ball_dir=[2,3,0,1][self.ball_dir] new_y = self.last_y else: self.terminal = True
if(not strike_toggle): self.strike = False
self.ball_x = new_x self.ball_y = new_y return r, self.terminal
# Query the current level of the difficulty ramp, difficulty does not ramp in this game, so return None def difficulty_ramp(self): return None
# Process the game-state into the 10x10xn state provided to the agent and return def state(self): state = np.zeros((10,10,len(self.channels)),dtype=bool) state[self.ball_y,self.ball_x,self.channels['ball']] = 1 state[9,self.pos, self.channels['paddle']] = 1 state[self.last_y,self.last_x,self.channels['trail']] = 1 state[:,:,self.channels['brick']] = self.brick_map return state
# Reset to start state for new episode def reset(self): self.ball_y = 3 ball_start = self.random.randint(2) self.ball_x, self.ball_dir = [(0,2),(9,3)][ball_start] self.pos = 4 self.brick_map = np.zeros((10,10)) self.brick_map[1:4,:] = 1 self.strike = False self.last_x = self.ball_x self.last_y = self.ball_y self.terminal = False
# Dimensionality of the game-state (10x10xn) def state_shape(self): return [10,10,len(self.channels)]
# Subset of actions that actually have a unique impact in this environment def minimal_action_set(self): minimal_actions = ['n','l','r'] return [self.action_map.index(x) for x in minimal_actions]W1[12, 0] = 2.5 # one entry: row 12, column 0W1[0:6, 3] = [1, 0, -1, 0, 2, 0] # explicit values along a range (as many values as entries)b1[0:4] = -1 # fill a range (a:b is half-open; ':' alone = the whole axis)W1[3:600:60, 0] = [0, 0, 0, 0, 0, 0, 0, 0, 1, 1] # column 0 of channel 3, rows 0..9 of the grid -> unit 0W2[0:2, 0:3] = [[1, -1, 0.5], [0, 2, 0]] # a 2-D list for a 2-D block (here 2 x 3)W3[:, 2] *= 2 # scale (also += and -=)# Units 0..3: descending-right, descending-left, ascending-right, ascending-left.# Subtract paddle position from each target.W1[540:600:6, 0] = [0, -0.02, -0.04, -0.06, -0.08, -0.10, -0.12, -0.14, -0.16, -0.18]W1[540:600:6, 1] = [0, -0.02, -0.04, -0.06, -0.08, -0.10, -0.12, -0.14, -0.16, -0.18]W1[540:600:6, 2] = [0, -0.02, -0.04, -0.06, -0.08, -0.10, -0.12, -0.14, -0.16, -0.18]W1[540:600:6, 3] = [0, -0.02, -0.04, -0.06, -0.08, -0.10, -0.12, -0.14, -0.16, -0.18]
# Ball row 0.W1[3:60:6, 0] = [0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04]W1[3:60:6, 1] = [0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02]W1[3:60:6, 2] = [0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02]W1[3:60:6, 3] = [0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0]
# Ball row 1.W1[63:120:6, 0] = [0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06]W1[63:120:6, 1] = [0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04]W1[63:120:6, 2] = [0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0]W1[63:120:6, 3] = [0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0]
# Ball row 2.W1[123:180:6, 0] = [0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08]W1[123:180:6, 1] = [0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04, 0.06]W1[123:180:6, 2] = [0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06]W1[123:180:6, 3] = [0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04]
# Ball row 3.W1[183:240:6, 0] = [0.10, 0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10]W1[183:240:6, 1] = [0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04, 0.06, 0.08]W1[183:240:6, 2] = [0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08]W1[183:240:6, 3] = [0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04, 0.06]
# Ball row 4.W1[243:300:6, 0] = [0.08, 0.10, 0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12]W1[243:300:6, 1] = [0.06, 0.04, 0.02, 0, 0, 0.02, 0.04, 0.06, 0.08, 0.10]W1[243:300:6, 2] = [0.10, 0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10]W1[243:300:6, 3] = [0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04, 0.06, 0.08]
# Ball row 5.W1[303:360:6, 0] = [0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14]W1[303:360:6, 1] = [0.04, 0.02, 0, 0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12]W1[303:360:6, 2] = [0.12, 0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08]W1[303:360:6, 3] = [0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04, 0.06]
# Ball row 6: descending targets become catch-interval midpoints.W1[363:420:6, 0] = [0.05, 0.07, 0.09, 0.11, 0.13, 0.15, 0.17, 0.18, 0.17, 0.15]W1[363:420:6, 1] = [0.03, 0.01, 0, 0.01, 0.03, 0.05, 0.07, 0.09, 0.11, 0.13]W1[363:420:6, 2] = [0.14, 0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06]W1[363:420:6, 3] = [0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02, 0.04]
# Ball row 7.W1[423:480:6, 0] = [0.03, 0.05, 0.07, 0.09, 0.11, 0.13, 0.15, 0.17, 0.18, 0.17]W1[423:480:6, 1] = [0.01, 0, 0.01, 0.03, 0.05, 0.07, 0.09, 0.11, 0.13, 0.15]W1[423:480:6, 2] = [0.16, 0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04]W1[423:480:6, 3] = [0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0, 0.02]
# Ball row 8.W1[483:540:6, 0] = [0.01, 0.03, 0.05, 0.07, 0.09, 0.11, 0.13, 0.15, 0.17, 0.18]W1[483:540:6, 1] = [0, 0.01, 0.03, 0.05, 0.07, 0.09, 0.11, 0.13, 0.15, 0.17]W1[483:540:6, 2] = [0.18, 0.18, 0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02]W1[483:540:6, 3] = [0.16, 0.14, 0.12, 0.10, 0.08, 0.06, 0.04, 0.02, 0, 0]
# Terminal-row fallback.W1[543:600:6, 0] = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18]W1[543:600:6, 1] = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18]W1[543:600:6, 2] = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18]W1[543:600:6, 3] = [0, 0.02, 0.04, 0.06, 0.08, 0.10, 0.12, 0.14, 0.16, 0.18]
# Unit 4: horizontal direction, including stationary wall-reflection frames.W1[3:600:60, 4] = 10W1[9:600:60, 4] = 20W1[15:600:60, 4] = 40W1[21:600:60, 4] = 60W1[27:600:60, 4] = 80W1[33:600:60, 4] = 100W1[39:600:60, 4] = 120W1[45:600:60, 4] = 140W1[51:600:60, 4] = 160W1[57:600:60, 4] = 170W1[10:600:60, 4] = -20W1[16:600:60, 4] = -40W1[22:600:60, 4] = -60W1[28:600:60, 4] = -80W1[34:600:60, 4] = -100W1[40:600:60, 4] = -120W1[46:600:60, 4] = -140W1[52:600:60, 4] = -160W1[58:600:60, 4] = -180
# Unit 5: descending versus ascending; a stationary row-8 frame means paddle bounce.b1[5] = 5W1[63:120:6, 5] = 10W1[123:180:6, 5] = 20W1[183:240:6, 5] = 30W1[243:300:6, 5] = 40W1[303:360:6, 5] = 50W1[363:420:6, 5] = 60W1[423:480:6, 5] = 70W1[483:540:6, 5] = 70W1[543:600:6, 5] = 90W1[64:120:6, 5] = -10W1[124:180:6, 5] = -20W1[184:240:6, 5] = -30W1[244:300:6, 5] = -40W1[304:360:6, 5] = -50W1[364:420:6, 5] = -60W1[424:480:6, 5] = -70W1[484:540:6, 5] = -80W1[544:600:6, 5] = -90
# Four context-gated pairs of left/right comparisons.W2[0, 0:2] = [-1000, 1000]W2[1, 2:4] = [-1000, 1000]W2[2, 4:6] = [-1000, 1000]W2[3, 6:8] = [-1000, 1000]W2[4, 0:8] = [200, 200, -200, -200, 200, 200, -200, -200]W2[5, 0:8] = [200, 200, 200, 200, -200, -200, -200, -200]b2[0:8] = -415
# Near-deterministic movement when needed; otherwise no-op.W3[0:8:2, 1] = 30W3[1:8:2, 3] = 30b3[:] = [0, 90, -100, 90, -100, -100]