From 0868868bab8f5df831db6aaf04301c9fc9244ea5 Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Fri, 14 Oct 2022 16:45:48 -0700 Subject: [PATCH 01/10] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a66cc4e..34cc37b 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # MapDatasetGenerator Generate and load dataset of road network maps. -# Quick start + # Installation from pip From e7cab84e3f77c32b489c396c45cec0ebdfcd5013 Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Fri, 14 Oct 2022 16:51:53 -0700 Subject: [PATCH 02/10] Update maps.py --- src/mapdataset/lib/maps.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index 2ef8096..0167066 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -73,10 +73,7 @@ def __len__(self): def __getitem__(self, idx): sample = (self.samples[idx], self.samples[idx], self.samples[idx]) - return torch.from_numpy(np.array(sample)).unsqueeze(0) - #flat = torch.from_numpy(np.array(sample)).view(-1) - #flat = flat[self.perm].float() - #return flat + return torch.from_numpy(sample).unsqueeze(0) def add(self, mapReader): for i in range(0, mapReader.size[0] - self.patch_size[0] + 1, self.stride): From 290c981414440576e601d87b79f8174cb94602d1 Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Fri, 14 Oct 2022 16:52:55 -0700 Subject: [PATCH 03/10] Update maps.py --- src/mapdataset/lib/maps.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index 0167066..0ae2acb 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -82,6 +82,10 @@ def add(self, mapReader): (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1 for y in range(self.patch_size[1])] for x in range(self.patch_size[0])]) + + def loadPrecomputed(self, patchDirectory): + #TODO + pass #Generate image patches and write to data/output directory def generate_patches(self, mapReader, image_groups=3, outDirectory=None): From 2931de60446667ec6766de14466919d1ee1f7562 Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Fri, 14 Oct 2022 16:55:14 -0700 Subject: [PATCH 04/10] Update maps.py --- src/mapdataset/lib/maps.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index 0ae2acb..7966967 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -76,12 +76,10 @@ def __getitem__(self, idx): return torch.from_numpy(sample).unsqueeze(0) def add(self, mapReader): + mapReader.standardize(self.converter) for i in range(0, mapReader.size[0] - self.patch_size[0] + 1, self.stride): for j in range(0, mapReader.size[1] - self.patch_size[1] + 1, self.stride): - self.samples.append([[ - (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1 - for y in range(self.patch_size[1])] - for x in range(self.patch_size[0])]) + self.samples.append(self.extractSample(mapReader, topLeft=(i, j))) def loadPrecomputed(self, patchDirectory): #TODO From cb73b203b1e6194509f7be024877de5e2e5557ed Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Fri, 14 Oct 2022 16:58:17 -0700 Subject: [PATCH 05/10] 2d mode --- ChangeLog.txt | 3 ++- src/mapdataset/lib/maps.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ChangeLog.txt b/ChangeLog.txt index e7c3b7d..f0e2500 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -4,4 +4,5 @@ Oct 14, 2022 3. Refactored ImageGroupReader 4. generate_patches normalizes the reader data once. 5. window_size -> patch_size -6. step_size -> stride \ No newline at end of file +6. step_size -> stride +7. 2d mode added. \ No newline at end of file diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index 7966967..ef1ff21 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -53,7 +53,7 @@ def get_char(self, layer): # Modifications - Ishaan, Muktadir class MapsDataset(Dataset): - def __init__(self, patch_size, stride, sample_group_size, converter, outputDir="../data/output"): + def __init__(self, patch_size, stride, sample_group_size, converter, outputDir="../data/output", mode='2d'): self.outputDir = outputDir self.char_size = converter.char_size self.converter = converter @@ -63,17 +63,20 @@ def __init__(self, patch_size, stride, sample_group_size, converter, outputDir=" self.samples = [] self.block_size = self.patch_size[0] * self.patch_size[1] - 1 - os.makedirs(self.outputDir, exist_ok=True) - - + self.mode = mode + os.makedirs(self.outputDir, exist_ok=True) def __len__(self): return len(self.samples) def __getitem__(self, idx): - sample = (self.samples[idx], self.samples[idx], self.samples[idx]) - return torch.from_numpy(sample).unsqueeze(0) + if self.mode =='2d': + sample = (self.samples[idx]) # 1 channel + return torch.from_numpy(np.array(sample)).unsqueeze(0).unsqueeze(0) + else: + sample = (self.samples[idx], self.samples[idx], self.samples[idx]) # 3 channels + return torch.from_numpy(sample).unsqueeze(0) def add(self, mapReader): mapReader.standardize(self.converter) From f6a7eb4534a30b853b7d55695a266b7c120638be Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Wed, 26 Oct 2022 16:06:45 -0700 Subject: [PATCH 06/10] loading from precomputed --- test.py => generate_patches.py | 30 +++++++++--------- read.py => read_patches.py | 2 +- src/mapdataset/lib/maps.py | 58 ++++++++++++++++++++++++++-------- src/{mapdataset => }/read.py | 0 src/{mapdataset => }/run.py | 0 5 files changed, 61 insertions(+), 29 deletions(-) rename test.py => generate_patches.py (60%) rename read.py => read_patches.py (90%) rename src/{mapdataset => }/read.py (100%) rename src/{mapdataset => }/run.py (100%) diff --git a/test.py b/generate_patches.py similarity index 60% rename from test.py rename to generate_patches.py index a921e44..2dd0e8c 100644 --- a/test.py +++ b/generate_patches.py @@ -29,18 +29,18 @@ mapsDataset.generate_patches(sfMap) #This will generate dill files which contain the saved sample lists. -dillFolder = "./data/output/SF_Layered/32x32/stride-10" -nGroups = 0 -# Iterate directory -for path in os.listdir(dillFolder): - # check if current path is a file - if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"): - nGroups += 1 - -for i in range(nGroups): - reader = ImageGroupReader(dillFolder) - data = reader.load_group(groupNo=i) - patchImgArray = reader.asImg(data[0]) - im = Image.fromarray(patchImgArray) - path = os.path.join(dillFolder, f"{i}-0.png") - im.save(path) +# dillFolder = "./data/output/SF_Layered/32x32/stride-10" +# nGroups = 0 +# # Iterate directory +# for path in os.listdir(dillFolder): +# # check if current path is a file +# if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"): +# nGroups += 1 + +# for i in range(nGroups): +# reader = ImageGroupReader(dillFolder) +# data = reader.load_group(groupNo=i) +# patchImgArray = reader.asImg(data[0]) +# im = Image.fromarray(patchImgArray) +# path = os.path.join(dillFolder, f"{i}-0.png") +# im.save(path) diff --git a/read.py b/read_patches.py similarity index 90% rename from read.py rename to read_patches.py index 329434c..500044b 100644 --- a/read.py +++ b/read_patches.py @@ -7,7 +7,7 @@ from mapdataset import ImageGroupReader -dillFolder = "./data/output/SF_Layered/32x32/stride-10" +dillFolder = "./data/output/SF_Layered/32x32/group-1280-stride-10" nGroups = 0 # Iterate directory for path in os.listdir(dillFolder): diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index ef1ff21..16f7aa9 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -61,16 +61,27 @@ def __init__(self, patch_size, stride, sample_group_size, converter, outputDir=" self.stride = stride self.sample_group_size = sample_group_size self.samples = [] + self.sampleGroupFiles = [] + self.currentGroup = None + self.currentGroupNo = None self.block_size = self.patch_size[0] * self.patch_size[1] - 1 self.mode = mode os.makedirs(self.outputDir, exist_ok=True) + self.__precomputedPatches = False + def __len__(self): + if self.__precomputedPatches: + return len(self.sampleGroupFiles * self.sample_group_size) return len(self.samples) def __getitem__(self, idx): + + if self.__precomputedPatches: + return self.__getPreComputedSample__(idx) + if self.mode =='2d': sample = (self.samples[idx]) # 1 channel return torch.from_numpy(np.array(sample)).unsqueeze(0).unsqueeze(0) @@ -84,12 +95,41 @@ def add(self, mapReader): for j in range(0, mapReader.size[1] - self.patch_size[1] + 1, self.stride): self.samples.append(self.extractSample(mapReader, topLeft=(i, j))) - def loadPrecomputed(self, patchDirectory): + + def loadPatches(self, patchDirectory): #TODO + + self.sampleFiles = [os.path.join(patchDirectory, f) for f in os.listdir(patchDirectory) if os.path.isfile(os.path.join(patchDirectory, f))] + logging.info(f"Loading {len(self.sampleFiles)} patches from {patchDirectory}") + self.__precomputedPatches = True + pass - #Generate image patches and write to data/output directory - def generate_patches(self, mapReader, image_groups=3, outDirectory=None): + #region Generate image patches and write to data/output directory + + def __getPreComputedSample__(self, idx): + # file = self.sampleFiles[idx] + # find group + groupNo = idx // self.sample_group_size + if self.currentGroupNo != groupNo: + # we don't have the group in memory + with open(self.sampleGroupFiles[groupNo]) as f: + self.currentGroup = dill.load(f) + + relativeIdx = idx % self.sample_group_size + return self.currentGroup[relativeIdx] + + + + + def __createDirectoryForPatches(self, mapReader, outDirectory=None): + + if outDirectory is None: + outDirectory = os.path.join(self.outputDir, mapReader.mapName, f"{self.patch_size[0]}x{self.patch_size[1]}", f"group-{self.sample_group_size}-stride-{self.stride}") + os.makedirs(outDirectory, exist_ok=True) + return outDirectory + + def generate_patches(self, mapReader, outDirectory=None): """_summary_ Args: @@ -99,9 +139,7 @@ def generate_patches(self, mapReader, image_groups=3, outDirectory=None): mapReader.standardize(converter=self.converter) - if outDirectory is None: - outDirectory = os.path.join(self.outputDir, mapReader.mapName, f"{self.patch_size[0]}x{self.patch_size[1]}", f"stride-{self.stride}") - os.makedirs(outDirectory, exist_ok=True) + outDirectory = self.__createDirectoryForPatches(mapReader, outDirectory) img_group_number = 0 for i in range(0, mapReader.size[0] - self.patch_size[0] + 1, self.stride): @@ -124,13 +162,7 @@ def generate_patches(self, mapReader, image_groups=3, outDirectory=None): def extractSample(self, mapReader, topLeft): i = topLeft[0] j = topLeft[1] - # sample = [ - # [ - # (self.converter.get_char(mapReader.data[i + x][j + y]) / (len(self.converter.char_groups) - 1)) * -2 + 1 # TODO this conversion should be done once in the original data instead of patches. - # for y in range(self.patch_size[1]) - # ] - # for x in range(self.patch_size[0]) - # ] + sample = [ [ mapReader.data[i + x][j + y] diff --git a/src/mapdataset/read.py b/src/read.py similarity index 100% rename from src/mapdataset/read.py rename to src/read.py diff --git a/src/mapdataset/run.py b/src/run.py similarity index 100% rename from src/mapdataset/run.py rename to src/run.py From 515547b934d371528fe5ee795c9c1e279d4a3bcf Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Wed, 26 Oct 2022 16:28:25 -0700 Subject: [PATCH 07/10] group file name read order fixed --- read_patches.py | 59 +++++++++++++++++++++++++++----------- src/mapdataset/lib/maps.py | 19 ++++++++---- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/read_patches.py b/read_patches.py index 500044b..1493533 100644 --- a/read_patches.py +++ b/read_patches.py @@ -1,24 +1,49 @@ # Script to read dill data objects as numpy arrays. -import dill -import numpy as np from PIL import Image import os +import sys +import logging +from random import randint -from mapdataset import ImageGroupReader +root = logging.getLogger() +root.setLevel(logging.DEBUG) +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +handler.setFormatter(formatter) +root.addHandler(handler) -dillFolder = "./data/output/SF_Layered/32x32/group-1280-stride-10" -nGroups = 0 -# Iterate directory -for path in os.listdir(dillFolder): - # check if current path is a file - if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"): - nGroups += 1 +from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader -for i in range(nGroups): - reader = ImageGroupReader(dillFolder) - data = reader.load_group(groupNo=i) - patchImgArray = reader.asImg(data[0]) - im = Image.fromarray(patchImgArray) - path = os.path.join(dillFolder, f"{i}-0.png") - im.save(path) + + +# dillFolder = "./data/output/SF_Layered/32x32/group-1280-stride-10" +# nGroups = 0 +# # Iterate directory +# for path in os.listdir(dillFolder): +# # check if current path is a file +# if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"): +# nGroups += 1 + +# for i in range(nGroups): +# reader = ImageGroupReader(dillFolder) +# data = reader.load_group(groupNo=i) +# patchImgArray = reader.asImg(data[0]) +# im = Image.fromarray(patchImgArray) +# path = os.path.join(dillFolder, f"{i}-0.png") +# im.save(path) + +mapsDataset = MapsDataset( + patch_size=(32, 32), + stride=10, + sample_group_size=1280, + converter=single_layer_converter, + outputDir="./data/output" + ) + +mapsDataset.loadPatches("./data/output/SF_Layered/32x32/group-1280-stride-10") +patchNo = randint(0, len(mapsDataset)) +logging.info(f"reading patch {patchNo}") +patch = mapsDataset[patchNo] + \ No newline at end of file diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index 16f7aa9..a7c0b75 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -99,11 +99,20 @@ def add(self, mapReader): def loadPatches(self, patchDirectory): #TODO - self.sampleFiles = [os.path.join(patchDirectory, f) for f in os.listdir(patchDirectory) if os.path.isfile(os.path.join(patchDirectory, f))] - logging.info(f"Loading {len(self.sampleFiles)} patches from {patchDirectory}") + groupFiles = [os.path.join(patchDirectory, f) + for f in os.listdir(patchDirectory) + if os.path.isfile(os.path.join(patchDirectory, f)) and + f.endswith(".dill")] + self.sampleGroupFiles = sorted(groupFiles, key=lambda f: self.filenameComparator(f)) + logging.info(f"Loading {len(self.sampleGroupFiles) * self.sample_group_size} patches from {patchDirectory}") self.__precomputedPatches = True pass + + + def filenameComparator(self, f): + _, tail = os.path.split(f) + return int(tail.split(".")[0]) #region Generate image patches and write to data/output directory @@ -113,15 +122,15 @@ def __getPreComputedSample__(self, idx): groupNo = idx // self.sample_group_size if self.currentGroupNo != groupNo: # we don't have the group in memory - with open(self.sampleGroupFiles[groupNo]) as f: + logging.debug(f"Opening group file {self.sampleGroupFiles[groupNo]}") + with open(self.sampleGroupFiles[groupNo], "rb") as f: self.currentGroup = dill.load(f) relativeIdx = idx % self.sample_group_size + logging.debug(f"patch {idx} is in group {groupNo} with relativeIdx {relativeIdx}") return self.currentGroup[relativeIdx] - - def __createDirectoryForPatches(self, mapReader, outDirectory=None): if outDirectory is None: From d6504e8e76cd36e650b1de2c0c7b17d934818466 Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Wed, 26 Oct 2022 16:38:00 -0700 Subject: [PATCH 08/10] need to save patches in pytorch image format. right now it's in PIL format --- read_patches.py | 9 +++++++-- src/mapdataset/lib/ImageUtils.py | 17 +++++++++++++++++ src/mapdataset/lib/__init__.py | 1 + 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 src/mapdataset/lib/ImageUtils.py diff --git a/read_patches.py b/read_patches.py index 1493533..b4c2438 100644 --- a/read_patches.py +++ b/read_patches.py @@ -14,11 +14,12 @@ handler.setFormatter(formatter) root.addHandler(handler) -from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader +from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader, ImageUtils -# dillFolder = "./data/output/SF_Layered/32x32/group-1280-stride-10" + +dillFolder = "./data/output/SF_Layered/32x32/group-1280-stride-10" # nGroups = 0 # # Iterate directory # for path in os.listdir(dillFolder): @@ -46,4 +47,8 @@ patchNo = randint(0, len(mapsDataset)) logging.info(f"reading patch {patchNo}") patch = mapsDataset[patchNo] + +im = ImageUtils.PILPatchToPILImg(patch) +path = os.path.join(dillFolder, f"{patchNo}.png") +im.save(path) \ No newline at end of file diff --git a/src/mapdataset/lib/ImageUtils.py b/src/mapdataset/lib/ImageUtils.py new file mode 100644 index 0000000..bdbe409 --- /dev/null +++ b/src/mapdataset/lib/ImageUtils.py @@ -0,0 +1,17 @@ +import numpy as np +from PIL import Image + +class ImageUtils: + + @staticmethod + def PILPatchToPILImg(patch): + data = np.clip((patch + 1) / 2, 0, 1) * 255 + data = data.astype(np.uint8) + return Image.fromarray(data) + + @staticmethod + def TorchPatchToPILImg(patch): + # TODO fix dimensions + data = np.clip((patch + 1) / 2, 0, 1) * 255 + data = data.astype(np.uint8) + return Image.fromarray(data) \ No newline at end of file diff --git a/src/mapdataset/lib/__init__.py b/src/mapdataset/lib/__init__.py index 780224c..1f28ab0 100644 --- a/src/mapdataset/lib/__init__.py +++ b/src/mapdataset/lib/__init__.py @@ -1,2 +1,3 @@ from .ImageGroupReader import ImageGroupReader +from .ImageUtils import ImageUtils from .maps import * \ No newline at end of file From b3977079cdd3049c8b6798ae6668a71e08042c1a Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Fri, 4 Nov 2022 13:55:15 -0700 Subject: [PATCH 09/10] pathes now have tensor dimensions --- read_patches.py | 4 +++- src/mapdataset/lib/ImageUtils.py | 25 ++++++++++++++++++++++++- src/mapdataset/lib/maps.py | 14 +++++++++----- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/read_patches.py b/read_patches.py index b4c2438..25cf672 100644 --- a/read_patches.py +++ b/read_patches.py @@ -48,7 +48,9 @@ logging.info(f"reading patch {patchNo}") patch = mapsDataset[patchNo] -im = ImageUtils.PILPatchToPILImg(patch) +# im = ImageUtils.PILPatchToPILImg(patch) +# im = ImageUtils.PILPatchToPILImg(patch) +im = ImageUtils.TorchNpPatchToPILImgGray(patch) path = os.path.join(dillFolder, f"{patchNo}.png") im.save(path) \ No newline at end of file diff --git a/src/mapdataset/lib/ImageUtils.py b/src/mapdataset/lib/ImageUtils.py index bdbe409..cc2c341 100644 --- a/src/mapdataset/lib/ImageUtils.py +++ b/src/mapdataset/lib/ImageUtils.py @@ -1,5 +1,6 @@ import numpy as np from PIL import Image +import torch class ImageUtils: @@ -14,4 +15,26 @@ def TorchPatchToPILImg(patch): # TODO fix dimensions data = np.clip((patch + 1) / 2, 0, 1) * 255 data = data.astype(np.uint8) - return Image.fromarray(data) \ No newline at end of file + return Image.fromarray(data) + + @staticmethod + def TorchPatchToPILImg(t): + return Image.fromarray(np.array(((t.squeeze(0).permute(1, 2, 0)+1)/2).clip(0, 1)*255).astype(np.uint8)) + + + @staticmethod + def TorchPatchToPILImgGray(t): + return Image.fromarray(np.array(((t.squeeze()+1)/2).clip(0, 1)*255).astype(np.uint8), "L") + + @staticmethod + def TorchNpPatchToPILImgGray(t): + # both tensor and numpy arrays have squeeze method + return ImageUtils.TorchPatchToPILImgGray(t) + + @staticmethod + def PILImgToTorch(im): # batch with 1. Standardized into (-1, 1). (h, w, c) - > (c, h, w) + return torch.tensor(np.array(im.convert('RGB'))/255).permute(2, 0, 1).unsqueeze(0) * 2 - 1 + + @staticmethod + def PILGrayToTorch(im): # batch with 1. Standardized into (-1, 1). (h, w, c) - > (c, h, w) gray has no extra color channel + return torch.tensor(np.array(im.convert('L'))/255).unsqueeze(0) * 2 - 1 \ No newline at end of file diff --git a/src/mapdataset/lib/maps.py b/src/mapdataset/lib/maps.py index a7c0b75..956f09c 100644 --- a/src/mapdataset/lib/maps.py +++ b/src/mapdataset/lib/maps.py @@ -53,7 +53,7 @@ def get_char(self, layer): # Modifications - Ishaan, Muktadir class MapsDataset(Dataset): - def __init__(self, patch_size, stride, sample_group_size, converter, outputDir="../data/output", mode='2d'): + def __init__(self, patch_size, stride, sample_group_size, converter, outputDir="../data/output", mode='gray'): self.outputDir = outputDir self.char_size = converter.char_size self.converter = converter @@ -80,9 +80,9 @@ def __len__(self): def __getitem__(self, idx): if self.__precomputedPatches: - return self.__getPreComputedSample__(idx) + return torch.from_numpy(self.__getPreComputedSample__(idx)) - if self.mode =='2d': + if self.mode =='gray': sample = (self.samples[idx]) # 1 channel return torch.from_numpy(np.array(sample)).unsqueeze(0).unsqueeze(0) else: @@ -139,7 +139,7 @@ def __createDirectoryForPatches(self, mapReader, outDirectory=None): return outDirectory def generate_patches(self, mapReader, outDirectory=None): - """_summary_ + """patches already have tensor like shape ready to be trained. No need to unsqueeze them. Args: mapReader (MapReader): reader for a single big map! @@ -179,7 +179,11 @@ def extractSample(self, mapReader, topLeft): ] for x in range(self.patch_size[0]) ] - return np.asarray(sample) + imArr = np.asarray(sample) + if self.mode == "gray": + return np.expand_dims(np.expand_dims(imArr, axis=0), axis=0) + else: + return np.expand_dims(imArr, axis=0) def shuffle(self): From a54a4285f3a2b8b72259ef812d084d093b0db670 Mon Sep 17 00:00:00 2001 From: adhocmaster Date: Tue, 8 Nov 2022 14:11:47 -0800 Subject: [PATCH 10/10] version 0.0.2 --- README.md | 41 +++++++++++++++++++++++++---------------- pyproject.toml | 2 +- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 34cc37b..00a9b43 100644 --- a/README.md +++ b/README.md @@ -62,23 +62,32 @@ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(messag handler.setFormatter(formatter) root.addHandler(handler) -from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader +from mapdataset import ImageGroupReader, single_layer_converter, MapsDataset, MapReader, ImageUtils + +dillFolder = "./data/output/SF_Layered/32x32/group-1280-stride-10" + +mapsDataset = MapsDataset( + patch_size=(32, 32), + stride=10, + sample_group_size=1280, + converter=single_layer_converter, + outputDir="./data/output" + ) + +mapsDataset.loadPatches("./data/output/SF_Layered/32x32/group-1280-stride-10") +patchNo = randint(0, len(mapsDataset)) +logging.info(f"reading patch {patchNo}") +patch = mapsDataset[patchNo] -dillFolder = "./data/output/SF_Layered/32x32/stride-10" -nGroups = 0 -# Iterate directory -for path in os.listdir(dillFolder): - # check if current path is a file - if os.path.isfile(os.path.join(dillFolder, path)) and path.endswith(".dill"): - nGroups += 1 +im = ImageUtils.TorchNpPatchToPILImgGray(patch) +path = os.path.join(dillFolder, f"{patchNo}.png") +im.save(path) + +``` + +# Using for training -for i in range(nGroups): - reader = ImageGroupReader(dillFolder) - data = reader.load_group(groupNo=i) - patchImgArray = reader.asImg(data[0]) - im = Image.fromarray(patchImgArray) - path = os.path.join(dillFolder, f"{i}-0.png") - im.save(path) -``` \ No newline at end of file +1. Create patches if you already do not have them +2. Create a MapsDataset object and load patches. Now you can use the dataset object as a regular Pytorch dataset or use it with a Dataloader. diff --git a/pyproject.toml b/pyproject.toml index ae80e6f..0ecdd8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "MapDatasetGenerator" -version = "0.0.1" +version = "0.0.2" description = "Map dataset generator for learning map representations and generation" authors = ["Ishaan ", "Golam Md Muktadir "] readme = "README.md"