Skip to content

Commit

Permalink
Optimize canvas with work queue to limit memory usage
Browse files Browse the repository at this point in the history
  • Loading branch information
@DonoA
DonoA committed Jan 5, 2019
1 parent d715001 commit 3626e6e
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 43 deletions.
33 changes: 29 additions & 4 deletions canvas.py
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,43 @@
import sys, math

class WorldTask:
def __init__(self, location, new_state):
self.location = location
self.new_state = new_state

class Canvas:
def __init__(self, world):
def __init__(self, world, auto_commit=True):
self.world = world
self.work_queue = []
self.auto_commit = auto_commit

def brush(self, state):
self.state = state
self.state = state.clone()
return self

def commit(self):
region_work = {}
for task in self.work_queue:
chunk_cords = self.world._get_chunk(task.location)
region_cords = self.world._get_region_file(chunk_cords)
if region_cords not in region_work:
region_work[region_cords] = []
region_work[region_cords].append(task)

for chunk, work in region_work.items():
for task in work:
self.world.get_block(task.location).set_state(task.new_state)
self.world.flush()

def square(self, center, side_len):
strt = math.floor(side_len/2.0)
end = math.ceil(side_len/2.0)
for x in range(-strt, end):
for z in range(-strt, end):
blk = self.world.get_block((center[0] + x, center[1], center[2] + z))
blk.set_state(self.state.clone())
loc = (center[0] + x, center[1], center[2] + z)
self.work_queue.append(WorldTask(loc, self.state))
if self.auto_commit:
self.commit()

def disk(self, center, radius):
ss = radius+1
Expand All @@ -23,6 +46,8 @@ def disk(self, center, radius):
loc = (center[0] + x, center[1], center[2] + z)
if Canvas._dist(loc, center) <= (radius + 0.5):
self.world.get_block((center[0] + x, center[1], center[2] + z)).set_state(self.state)
if self.auto_commit:
self.commit()

def _dist(loc1, loc2):
dx = abs(loc1[0] - loc2[0]) ** 2
Expand Down
31 changes: 3 additions & 28 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,34 +5,9 @@
from materials import Material
from biomes import Biome

with World('A', save_location='/home/dallen/.minecraft/saves') as wrld:
with World('A', save_location='/home/dallen/.minecraft/saves', debug=True) as wrld:
print('World loaded!')
wrld.get_block((99, 95, -203)).set_state(Material.diamond_block)
wrld.get_block((99, 94, -203)).set_state(Material.coal_block)
wrld.get_block((99, 93, -203)).set_state(Material.gold_block)
wrld.get_block((99, 92, -203)).set_state(Material.iron_block)
wrld.get_block((99, 91, -203)).set_state(Material.glowstone)

wrld.get_block((98, 95, -203)).set_state(Material.diamond_ore)
wrld.get_block((98, 94, -203)).set_state(Material.coal_ore)
wrld.get_block((98, 93, -203)).set_state(Material.gold_ore)
wrld.get_block((98, 92, -203)).set_state(Material.iron_ore)
wrld.get_block((98, 91, -203)).set_state(Material.nether_bricks)

wrld.get_block((100, 95, -203)).set_state(Material.acacia_planks)
wrld.get_block((100, 94, -203)).set_state(Material.birch_planks)
wrld.get_block((100, 93, -203)).set_state(Material.dark_oak_planks)
wrld.get_block((100, 92, -203)).set_state(Material.jungle_planks)
wrld.get_block((100, 91, -203)).set_state(Material.spruce_planks)

wrld.get_block((101, 95, -203)).set_state(Material.stone)
wrld.get_block((101, 94, -203)).set_state(Material.andesite)
wrld.get_block((101, 93, -203)).set_state(Material.gravel)
wrld.get_block((101, 92, -203)).set_state(Material.gray_wool)
wrld.get_block((101, 91, -203)).set_state(Material.black_wool)

# print(blk)
# cv = Canvas(wrld)
# cv.brush(BlockState(Material.gold_block, {})).square((106, 95, -252), 100)
cv = wrld.get_canvas()
cv.brush(BlockState(Material.diamond_block, {})).square((106, 90, -252), 100)

print('Saved!')
29 changes: 18 additions & 11 deletions world.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import sys, math, nbt, gzip, zlib, stream, time, os
from biomes import Biome
from canvas import Canvas

class BlockState:
def __init__(self, name, props):
Expand Down Expand Up @@ -218,7 +219,8 @@ def __str__(self):
return "Chunk(" + str(self.xpos) + "," + str(self.zpos) + ")"

class World:
def __init__(self, file_name, save_location=''):
def __init__(self, file_name, save_location='', debug=False, read=True, write=True):
self.debug = debug
self.file_name = file_name
self.save_location = save_location
if not os.path.exists(save_location):
Expand All @@ -234,6 +236,10 @@ def __exit__(self, typ, val, trace):
if typ is None:
self.close()

def flush(self):
self.close()
self.chunks = {}

def close(self):
chunks_by_region = {}
for chunk_pos, chunk in self.chunks.items():
Expand Down Expand Up @@ -264,18 +270,19 @@ def close(self):
data = zlib.compress(strm.get_data())
datalen = len(data)
block_data_len = math.ceil((datalen + 5)/4096.0)*4096
# Constuct new data block
data = (datalen + 1).to_bytes(4, byteorder='big', signed=False) + \
(2).to_bytes(1, byteorder='big', signed=False) + \
data + \
(0).to_bytes(block_data_len - (datalen + 5), byteorder='big', signed=False)

# timestamps[((chunk.xpos % 32) + (chunk.zpos % 32) * 32)] = int(time.time())
timestamps[((chunk.xpos % 32) + (chunk.zpos % 32) * 32)] = int(time.time())

loc = locations[((chunk.xpos % 32) + (chunk.zpos % 32) * 32)]
original_sector_length = loc[1]
data_len_diff = block_data_len - original_sector_length
# if data_len_diff != 0:
# print(f'Danger: Diff is {data_len_diff}, shifting required!')
if data_len_diff != 0 and self.debug:
print(f'Danger: Diff is {data_len_diff}, shifting required!')

locations[((chunk.xpos % 32) + (chunk.zpos % 32) * 32)][1] = block_data_len

Expand All @@ -290,7 +297,8 @@ def close(self):

header_length = 2*4096
data_in_file[(loc[0] - header_length):(loc[0] + original_sector_length - header_length)] = data
print('Saving', chunk, 'With', {'loc': loc, 'new_len': datalen, 'old_len': chunk.orig_size, 'sector_len': block_data_len})
if self.debug:
print(f'Saving {chunk} with', {'loc': loc, 'new_len': datalen, 'old_len': chunk.orig_size, 'sector_len': block_data_len})

region.seek(0)

Expand Down Expand Up @@ -318,6 +326,9 @@ def get_chunk(self, chunk_pos):

return self.chunks[chunk_pos]

def get_canvas(self):
return Canvas(self)

def _load_chunk(self, chunk_pos):
with open(self.save_location + '/' + self.file_name + '/region/' + self._get_region_file(chunk_pos), mode='rb') as region:
locations = [[
Expand All @@ -328,18 +339,15 @@ def _load_chunk(self, chunk_pos):
timestamps = region.read(4096)

loc = locations[((chunk_pos[0] % 32) + (chunk_pos[1] % 32) * 32)]
# print(loc)

print('Loading', chunk_pos,'from', region.name)
if self.debug:
print('Loading', chunk_pos,'from', region.name)
chunk = self._load_binary_chunk_at(region, loc[0], loc[1])
self.chunks[chunk_pos] = chunk

def _load_binary_chunk_at(self, region_file, offset, max_size):
region_file.seek(offset)
datalen = int.from_bytes(region_file.read(4), byteorder='big', signed=False)
# print('Len', datalen, 'Max', max_size)
compr = region_file.read(1)
# print('Compr', compr)
# print(region_file.tell()-5, datalen)
decompressed = zlib.decompress(region_file.read(datalen-1))
data = nbt.parse_nbt(stream.InputStream(decompressed))
Expand All @@ -356,7 +364,6 @@ def _load_binary_chunk_at(self, region_file, offset, max_size):
def _get_region_file(self, chunk_pos):
return 'r.' + '.'.join([str(x) for x in self._get_region(chunk_pos)]) + '.mca'


def _get_chunk(self, block_pos):
return (math.floor(block_pos[0] / 16), math.floor(block_pos[2] / 16))

Expand Down

0 comments on commit 3626e6e

Please sign in to comment.