diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..a181dc7 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max_line_length = 180 diff --git a/.gitignore b/.gitignore index a2c1f4f..f88109f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,30 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class *.npy + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Installer logs +pip-log.txt + +# IDE project files +.idea +.vscode + +# Distribution / packaging +*_FSCT_output/ +*FSCT_output/ + +# Examples *.las !example.las model/training_history.csv -scripts/__pycache__ -.vscode -venv -.idea -docker -*FSCT_output \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b89c72a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +Main changes: +- Main file added to invoke code from project root path + +- Unused imports removed + +- The argparse module added (make easy to write user-friendly command-line interfaces) + +- FSCT main configuration moved to specific configuration files (config.ini) + +- Dockerized code with entrypoint to invoke executions from host \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5299915 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.9 + +RUN mkdir /app +RUN mkdir /datasets +WORKDIR /app + +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY data data +COPY model model +COPY scripts scripts +COPY tools tools +COPY config.ini . +COPY main.py . +COPY multiple_plot_centres_file_config.py . +COPY entrypoint.sh . + +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index cf885c9..1e147d0 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ it works (or doesn't), please let me know! If you have any difficulties or find any bugs, please get in touch and I will try to help you get it going. Suggestions for improvements are greatly appreciated. -If you do not have an Nvidia GPU, please set the ```use_CPU_only``` setting in ```run.py``` to True. +If you do not have an Nvidia GPU, please set the ```use_cpu_only``` setting in ```run.py``` to True. ## How to use @@ -54,6 +54,39 @@ this will contain the following outputs. Start with small plots containing at least some trees. The tree measurement code will currently cause an error if it finds no trees in the point cloud. +## Docker +1. Create docker volume to access from host +``` +docker volume create datasets +``` +2. Build your docker image +``` +docker build --rm -t fsct-image . +``` +### Run image (ensure rebuild docker image before running) +``` +docker run -i --rm \ + -v "$(pwd)/datasets:/datasets" \ + --name fsct \ + fsct [--additional --parameters --here] +``` +* Run docker container and get into shell +``` +docker run --rm -v ~/datasets:/datasets --name fsct -it fsct-image /bin/bash +``` + +* Run docker FSCT with arguments + +#### Example: +> docker run --rm -v ~/datasets:/datasets --name fsct fsct-image -f ~/datasets/mydataset/model.laz + +### Run Docker FSCT with GPU support +You need to install [Nvidia Docker](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker) +#### Example: +> docker run --rm -v $work_dir:/datasets --gpus all --name fsct fsct-image -f /datasets/$dataset/model.laz + + + ## FSCT Outputs ```Plot_Report.html``` and ```Plot_Report.md``` diff --git a/config.ini b/config.ini new file mode 100644 index 0000000..da023b7 --- /dev/null +++ b/config.ini @@ -0,0 +1,106 @@ +[FSCT_main_parameters] + +# [X, Y] Coordinates of the plot centre (metres). If "None", plot_centre is computed based on the point cloud bounding box. +plot_centre=None + +# Circular Plot options - Leave at 0 if not using. +# If 0 m, the plot is not cropped. Otherwise, the plot is cylindrically cropped from the plot centre with plot_radius + plot_radius_buffer. +plot_radius=0 +# See README. If non-zero, this is used for "Tree Aware Plot Cropping Mode". +plot_radius_buffer=0 + +# Set these appropriately for your hardware. +# You will get CUDA errors if this is too high, as you will run out of VRAM. This won't be an issue if running on CPU only. Must be >= 2. +batch_size=8 +# Number of CPU cores you want to use. If you run out of RAM, lower this. 0 means ALL cores. +num_cpu_cores=0 +# Set to True if you do not have an Nvidia GPU, or if you don't have enough vRAM. +use_cpu_only=True + +# Optional settings - Generally leave as they are. +# If your point cloud resolution is a bit low (and only if the stem segmentation is still reasonably accurate), try increasing this to 0.2. +slice_thickness=0.15 +# If your point cloud is really dense, you may get away with 0.1. +# The smaller this is, the better your results will be, however, this increases the run time. +slice_increment=0.05 + +# If you don't need the sorted stem points, turning this off speeds things up. +# Veg sorting is required for tree height measurement, but stem sorting isn't necessary for standard use. +sort_stems=1 + +# If the data contains noise above the canopy, you may wish to set this to the 98th percentile of height, otherwise leave it at 100. +height_percentile=100 +# A tree must have a cylinder measurement below this height above the DTM to be kept. This filters unsorted branches from being called individual trees. +tree_base_cutoff_height=5 +# Turn on if you would like a semantic and instance segmented point cloud. This mode will override the "sort_stems" setting if on. +generate_output_point_cloud=1 + +# If you activate "tree aware plot cropping mode", this function will use it. +# Any vegetation points below this height are considered to be understory and are not assigned to individual trees. +ground_veg_cutoff_height=3 +# Vegetation points can be, at most, this far away from a cylinder horizontally to be matched to a particular tree. +veg_sorting_range=1.5 +# Stem points can be, at most, this far away from a cylinder in 3D to be matched to a particular tree. +stem_sorting_range=1 +# Lowest height to measure diameter for taper output. +taper_measurement_height_min=0 +# Highest height to measure diameter for taper output. +taper_measurement_height_max=30 +# diameter measurement increment. +taper_measurement_height_increment=0.2 +# Cylinder measurements within +/- 0.5*taper_slice_thickness are used for taper measurement at a given height. The largest diameter is used. +taper_slice_thickness=0.4 +# Generally leave this on. Deletes the files used for segmentation after segmentation is finished. +delete_working_directory=True +# You may wish to turn it off if you want to re-run/modify the segmentation code so you don't need to run pre-processing every time. +# Will delete a number of non-essential outputs to reduce storage use. +minimise_output_size_mode=0 + +[FSCT_other_parameters] +# Don't change these unless you really understand what you are doing with them/are learning how the code works. +# These have been tuned to work on most high resolution forest point clouds without changing them, but you may be able +# to tune these better for your particular data. Almost everything here is a trade-off between different situations, so +# optimisation is not straight-forward. + +model_filename="model.pth" +# Dimensions of the sliding box used for semantic segmentation. +box_dimensions=[6, 6, 6] +# Overlap of the sliding box used for semantic segmentation. +box_overlap=[0.5, 0.5, 0.5] +# Minimum number of points for input to the model. Too few points and it becomes near impossible to accurately label them (though assuming vegetation class is the safest bet here). +min_points_per_box=1000 +# Maximum number of points for input to the model. The model may tolerate higher numbers if you decrease the batch size accordingly (to fit on the GPU), but this is not tested. +max_points_per_box=20000 +# Don't change +noise_class=0 +# Don't change +terrain_class=1 +# Don't change +vegetation_class=2 +# Don't change +cwd_class=3 +# Don't change +stem_class=4 +# Resolution of the DTM. +grid_resolution=0.5 +vegetation_coverage_resolution=0.2 +num_neighbours=5 +sorting_search_angle=20 +sorting_search_radius=1 +sorting_angle_tolerance=90 +max_search_radius=3 +max_search_angle=30 +# Used for HDBSCAN clustering step. Recommend not changing for general use. +min_cluster_size=30 +# During cleaning, this w +cleaned_measurement_radius=0.2 +# Generally leave this on, but you can turn off subsampling. +subsample=0 +# The point cloud will be subsampled such that the closest any 2 points can be is 0.01 m. +subsampling_min_spacing=0.01 +# Minimum valid Circuferential Completeness Index (CCI) for non-interpolated circle/cylinder fitting. Any measurements with CCI below this are deleted. +minimum_cci=0.3 +# Deletes any trees with fewer than 10 cylinders (before the cylinder interpolation step). +min_tree_cyls=10 +# Very ugly hack that can sometimes be useful on point clouds which are on the borderline of having not enough points to be functional with FSCT. Set to a positive integer. Point cloud will be copied this many times (with noise added) to artificially increase point density giving the segmentation model more points. +low_resolution_point_cloud_hack_mode=0 diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..86430a7 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo Executing FSCT wit arguments: "$@" +python main.py "$@" \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..67e0a67 --- /dev/null +++ b/main.py @@ -0,0 +1,37 @@ +import sys +from scripts import run +from scripts import run_with_multiple_plot_centres +import multiple_plot_centres_file_config + +if __name__ == '__main__': + """ + Choose one of the following or modify as needed. + Directory mode will find all .las files within a directory and sub directories but will ignore any .las files in + folders with "FSCT_output" in their names. + File mode will allow you to select multiple .las files within a directory. + Alternatively, you can just list the point cloud file paths. + If you have multiple point clouds and wish to enter plot coords for each, have a look at "run_with_multiple_plot_centres.py" + """ + opts = [opt for opt in sys.argv[1:] if opt.startswith("-")] + args = [arg for arg in sys.argv[1:] if not arg.startswith("-")] + mode = "-a" + type = run + + # If multiple or single processing + if "-m" in opts: + type = run_with_multiple_plot_centres + # args indicates multiple_plot_centres_file_config + if args is None: + args = multiple_plot_centres_file_config + # directory mode + if "-d" in opts: + mode = "-d" + # file mode + elif "-f" in opts: + mode = "-f" + # attended user file mode + elif "-a" in opts: + mode = "-a" + else: + raise SystemExit(f"Usage: {sys.argv[0]} (-c | -u | -l) ...") + type.exec(mode, args) diff --git a/multiple_plot_centres_file_config.py b/multiple_plot_centres_file_config.py new file mode 100644 index 0000000..dd00105 --- /dev/null +++ b/multiple_plot_centres_file_config.py @@ -0,0 +1,9 @@ +''' +This script is an example of how to provide multiple different plot centres with your input point clouds. +@args +[[*.las, [your_plot_centre_X_coord, your_plot_centre_Y_coord], your_plot_radius], [*.las,[X,Y], radius], [...], [...]] +''' +multiple_plot_centres_file_config = [ + ['your_point_cloud1.las', [0, 0], 100], + ['your_point_cloud2.las', [300, 200], 50], +] diff --git a/requirements.txt b/requirements.txt index 24a7243..146aa24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ Jinja2==3.1.2 joblib==1.1.0 kiwisolver==1.4.2 laspy==2.1.2 +lazrs==0.4.1 Markdown==3.3.7 MarkupSafe==2.1.1 matplotlib==3.5.2 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/combine_multiple_output_CSVs.py b/scripts/combine_multiple_output_CSVs.py index d08eb9a..0f84786 100644 --- a/scripts/combine_multiple_output_CSVs.py +++ b/scripts/combine_multiple_output_CSVs.py @@ -1,6 +1,6 @@ import pandas as pd -from run_tools import FSCT, directory_mode, file_mode +from scripts.run_tools import FSCT, directory_mode, file_mode def combine_multiple_output_CSVs(point_clouds_to_process, csv_file_to_combine): diff --git a/scripts/inference.py b/scripts/inference.py index 88e39f2..f4ef699 100644 --- a/scripts/inference.py +++ b/scripts/inference.py @@ -1,18 +1,15 @@ +import os from abc import ABC import torch -import torch_geometric from torch_geometric.data import Dataset, DataLoader, Data import numpy as np import glob import pandas as pd -from preprocessing import Preprocessing -from model import Net +from scripts.model import Net from sklearn.neighbors import NearestNeighbors -from scipy import spatial -import os import time -from tools import get_fsct_path -from tools import load_file, save_file +from scripts.tools import get_fsct_path +from scripts.tools import load_file, save_file import shutil import sys @@ -69,10 +66,9 @@ class SemanticSegmentation: def __init__(self, parameters): self.sem_seg_start_time = time.time() self.parameters = parameters - - if not self.parameters["use_CPU_only"]: - print("Is CUDA available?", torch.cuda.is_available()) - self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + if not self.parameters['use_cpu_only']: + print('Is CUDA available?', torch.cuda.is_available()) + self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') else: self.device = torch.device("cpu") @@ -98,7 +94,7 @@ def inference(self): test_loader = DataLoader(test_dataset, batch_size=self.parameters["batch_size"], shuffle=False, num_workers=0) model = Net(num_classes=4).to(self.device) - if self.parameters["use_CPU_only"]: + if self.parameters["use_cpu_only"]: model.load_state_dict( torch.load( get_fsct_path("model") + "/" + self.parameters["model_filename"], diff --git a/scripts/measure.py b/scripts/measure.py index eb9b08c..f0d0682 100644 --- a/scripts/measure.py +++ b/scripts/measure.py @@ -1,41 +1,28 @@ import math -import sys from copy import deepcopy from multiprocessing import get_context import networkx as nx import numpy as np import pandas as pd import os +import time from scipy import spatial # TODO Test if sklearn kdtree is faster. -from scipy.spatial import ConvexHull -from scipy.interpolate import CubicSpline from scipy.interpolate import griddata -from skimage.measure import LineModelND, CircleModel, ransac -from sklearn.cluster import DBSCAN +from skimage.measure import CircleModel, ransac from sklearn.neighbors import NearestNeighbors -from sklearn.linear_model import RANSACRegressor -from sklearn.preprocessing import PolynomialFeatures -from sklearn.pipeline import make_pipeline -from tools import ( +from skspatial.objects import Plane +from sklearn.neighbors import BallTree +from scripts.tools import ( get_fsct_path, load_file, save_file, low_resolution_hack_mode, - subsample_point_cloud, - clustering, cluster_hdbscan, cluster_dbscan, get_heights_above_DTM, - subsample, get_taper, ) -from fsct_exceptions import DataQualityError -import time -import hdbscan -from skspatial.objects import Plane -import warnings -import scipy -from sklearn.neighbors import BallTree +from scripts.fsct_exceptions import DataQualityError class MeasureTree: @@ -394,7 +381,7 @@ def criteria_check(cyl1, cyl2, angle_tolerance, search_angle): sorted_points = np.vstack((sorted_points, current_point)) unsorted_points = np.vstack( - (unsorted_points[:current_point_index], unsorted_points[current_point_index + 1 :]) + (unsorted_points[:current_point_index], unsorted_points[current_point_index + 1:]) ) kdtree = spatial.cKDTree(unsorted_points[:, :3], leafsize=1000) results = kdtree.query_ball_point(np.atleast_2d(current_point)[:, :3], r=distance_tolerance)[0] @@ -808,6 +795,7 @@ def within_search_cone(cls, normal1, vector1_2, search_angle): return False def run_measurement_extraction(self): + taper_array = [] skeleton_array = np.zeros((0, 3)) cluster_array = np.zeros((0, 6)) slice_heights = np.linspace( @@ -911,8 +899,8 @@ def run_measurement_extraction(self): print("\r", max_j, "/", max_j, end="") print("\nDone\n") - print("Deleting cyls with CCI less than:", self.parameters["minimum_CCI"]) - full_cyl_array = full_cyl_array[full_cyl_array[:, self.cyl_dict["CCI"]] >= self.parameters["minimum_CCI"]] + print("Deleting cyls with CCI less than:", self.parameters["minimum_cci"]) + full_cyl_array = full_cyl_array[full_cyl_array[:, self.cyl_dict["CCI"]] >= self.parameters["minimum_cci"]] # cyl_array = [x,y,z,nx,ny,nz,r,CCI,branch_id,tree_id,segment_volume,parent_branch_id] print("Saving cylinder array...") diff --git a/scripts/model.py b/scripts/model.py index 49ced5c..3eb454f 100644 --- a/scripts/model.py +++ b/scripts/model.py @@ -1,86 +1,32 @@ -import torch -import torch.nn.functional as F -from torch_geometric.nn import knn_interpolate -from torch.nn import Sequential as Seq, Linear as Lin, ReLU, BatchNorm1d as BN -from torch_geometric.nn import PointConv, fps, radius, global_max_pool - - -class SAModule(torch.nn.Module): - def __init__(self, ratio, r, NN): - super(SAModule, self).__init__() - self.ratio = ratio - self.r = r - self.conv = PointConv(NN) - - def forward(self, x, pos, batch): - idx = fps(pos, batch, ratio=self.ratio) - row, col = radius(pos, pos[idx], self.r, batch, batch[idx], max_num_neighbors=64) - edge_index = torch.stack([col, row], dim=0) - x = self.conv(x, (pos, pos[idx]), edge_index) - pos, batch = pos[idx], batch[idx] - return x, pos, batch - - -class GlobalSAModule(torch.nn.Module): - def __init__(self, NN): - super(GlobalSAModule, self).__init__() - self.NN = NN - - def forward(self, x, pos, batch): - x = self.NN(torch.cat([x, pos], dim=1)) - x = global_max_pool(x, batch) - pos = pos.new_zeros((x.size(0), 3)) - batch = torch.arange(x.size(0), device=batch.device) - return x, pos, batch - - -def MLP(channels, batch_norm=True): - return Seq(*[Seq(Lin(channels[i - 1], channels[i]), ReLU(), BN(channels[i])) for i in range(1, len(channels))]) - - -class FPModule(torch.nn.Module): - def __init__(self, k, NN): - super(FPModule, self).__init__() - self.k = k - self.NN = NN - - def forward(self, x, pos, batch, x_skip, pos_skip, batch_skip): - x = knn_interpolate(x, pos, pos_skip, batch, batch_skip, k=self.k) - if x_skip is not None: - x = torch.cat([x, x_skip], dim=1) - x = self.NN(x) - return x, pos_skip, batch_skip - - -class Net(torch.nn.Module): - def __init__(self, num_classes): - super(Net, self).__init__() - self.sa1_module = SAModule(0.1, 0.2, MLP([3, 128, 256, 512])) - self.sa2_module = SAModule(0.05, 0.4, MLP([512 + 3, 512, 1024, 1024])) - self.sa3_module = GlobalSAModule(MLP([1024 + 3, 1024, 2048, 2048])) - - self.fp3_module = FPModule(1, MLP([3072, 1024, 1024])) - self.fp2_module = FPModule(3, MLP([1536, 1024, 1024])) - self.fp1_module = FPModule(3, MLP([1024, 1024, 1024])) - - self.conv1 = torch.nn.Conv1d(1024, 1024, 1) - self.conv2 = torch.nn.Conv1d(1024, num_classes, 1) - self.drop1 = torch.nn.Dropout(0.5) - self.bn1 = torch.nn.BatchNorm1d(1024) - - def forward(self, data): - sa0_out = (data.x, data.pos, data.batch) - sa1_out = self.sa1_module(*sa0_out) - sa2_out = self.sa2_module(*sa1_out) - sa3_out = self.sa3_module(*sa2_out) - - fp3_out = self.fp3_module(*sa3_out, *sa2_out) - fp2_out = self.fp2_module(*fp3_out, *sa1_out) - x, _, _ = self.fp1_module(*fp2_out, *sa0_out) - - x = x.unsqueeze(dim=0) - x = x.permute(0, 2, 1) - x = self.drop1(F.relu(self.bn1(self.conv1(x)))) - x = self.conv2(x) - x = F.log_softmax(x, dim=1) - return x +# Don't change these unless you really understand what you are doing with them/are learning how the code works. +# These have been tuned to work on most high resolution forest point clouds without changing them, but you may be able +# to tune these better for your particular data. Almost everything here is a trade-off between different situations, so +# optimisation is not straight-forward. + +other_parameters = dict( + model_filename="model.pth", + box_dimensions=[6, 6, 6], # Dimensions of the sliding box used for semantic segmentation. + box_overlap=[0.5, 0.5, 0.5], # Overlap of the sliding box used for semantic segmentation. + min_points_per_box=1000, # Minimum number of points for input to the model. Too few points and it becomes near impossible to accurately label them (though assuming vegetation class is the safest bet here). + max_points_per_box=20000, # Maximum number of points for input to the model. The model may tolerate higher numbers if you decrease the batch size accordingly (to fit on the GPU), but this is not tested. + noise_class=0, # Don't change + terrain_class=1, # Don't change + vegetation_class=2, # Don't change + cwd_class=3, # Don't change + stem_class=4, # Don't change + grid_resolution=0.5, # Resolution of the DTM. + vegetation_coverage_resolution=0.2, + num_neighbours=5, + sorting_search_angle=20, + sorting_search_radius=1, + sorting_angle_tolerance=90, + max_search_radius=3, + max_search_angle=30, + min_cluster_size=30, # Used for HDBSCAN clustering step. Recommend not changing for general use. + cleaned_measurement_radius=0.2, # During cleaning, this w + subsample=0, # Generally leave this on, but you can turn off subsampling. + subsampling_min_spacing=0.01, # The point cloud will be subsampled such that the closest any 2 points can be is 0.01 m. + minimum_cci=0.3, # Minimum valid Circuferential Completeness Index (CCI) for non-interpolated circle/cylinder fitting. Any measurements with CCI below this are deleted. + min_tree_cyls=10, # Deletes any trees with fewer than 10 cylinders (before the cylinder interpolation step). + low_resolution_point_cloud_hack_mode=0, +) # Very ugly hack that can sometimes be useful on point clouds which are on the borderline of having not enough points to be functional with FSCT. Set to a positive integer. Point cloud will be copied this many times (with noise added) to artificially increase point density giving the segmentation model more points. diff --git a/scripts/other_parameters.py b/scripts/other_parameters.py index 665260c..3eb454f 100644 --- a/scripts/other_parameters.py +++ b/scripts/other_parameters.py @@ -26,7 +26,7 @@ cleaned_measurement_radius=0.2, # During cleaning, this w subsample=0, # Generally leave this on, but you can turn off subsampling. subsampling_min_spacing=0.01, # The point cloud will be subsampled such that the closest any 2 points can be is 0.01 m. - minimum_CCI=0.3, # Minimum valid Circuferential Completeness Index (CCI) for non-interpolated circle/cylinder fitting. Any measurements with CCI below this are deleted. + minimum_cci=0.3, # Minimum valid Circuferential Completeness Index (CCI) for non-interpolated circle/cylinder fitting. Any measurements with CCI below this are deleted. min_tree_cyls=10, # Deletes any trees with fewer than 10 cylinders (before the cylinder interpolation step). low_resolution_point_cloud_hack_mode=0, ) # Very ugly hack that can sometimes be useful on point clouds which are on the borderline of having not enough points to be functional with FSCT. Set to a positive integer. Point cloud will be copied this many times (with noise added) to artificially increase point density giving the segmentation model more points. diff --git a/scripts/post_segmentation_script.py b/scripts/post_segmentation_script.py index 380aa83..581e957 100644 --- a/scripts/post_segmentation_script.py +++ b/scripts/post_segmentation_script.py @@ -1,30 +1,11 @@ import numpy as np -from matplotlib import pyplot as plt -import matplotlib -from matplotlib.patches import Circle, PathPatch -from matplotlib import cm -from mpl_toolkits.mplot3d import Axes3D -from skimage.measure import LineModelND, CircleModel, ransac, EllipseModel -import mpl_toolkits.mplot3d.art3d as art3d -import math import pandas as pd -from scipy import stats, spatial +from scipy import spatial import time import warnings -from sklearn import metrics -from sklearn.preprocessing import StandardScaler -from copy import deepcopy -from skimage.measure import LineModelND, CircleModel, ransac -import glob -from scipy.spatial import ConvexHull, convex_hull_plot_2d -from scipy.spatial.distance import euclidean -from math import sin, cos, pi -import random import os -from sklearn.neighbors import NearestNeighbors -from tools import load_file, save_file, subsample_point_cloud, get_heights_above_DTM, cluster_dbscan -from scipy.interpolate import griddata -from fsct_exceptions import DataQualityError +from scripts.tools import load_file, save_file, get_heights_above_DTM +from scripts.fsct_exceptions import DataQualityError warnings.filterwarnings("ignore", category=RuntimeWarning) diff --git a/scripts/preprocessing.py b/scripts/preprocessing.py index f545184..23ad222 100644 --- a/scripts/preprocessing.py +++ b/scripts/preprocessing.py @@ -1,16 +1,10 @@ import numpy as np import time -import glob import random import pandas as pd -from copy import deepcopy -import matplotlib -from scipy import spatial -from sklearn.neighbors import NearestNeighbors import threading -from tools import load_file, save_file, make_folder_structure, subsample_point_cloud, low_resolution_hack_mode +from scripts.tools import load_file, save_file, make_folder_structure, subsample_point_cloud, low_resolution_hack_mode import os -from multiprocessing import get_context class Preprocessing: diff --git a/scripts/report_writer.py b/scripts/report_writer.py index d69950c..42b43af 100644 --- a/scripts/report_writer.py +++ b/scripts/report_writer.py @@ -3,12 +3,9 @@ from mdutils import Html import pandas as pd import numpy as np -from tools import load_file, subsample_point_cloud +from scripts.tools import load_file from matplotlib import pyplot as plt import os -from scipy.spatial import ConvexHull -from matplotlib import cm -from matplotlib.patches import Patch from matplotlib.lines import Line2D import warnings import shutil diff --git a/scripts/run.py b/scripts/run.py index 15de405..8ca9c3e 100644 --- a/scripts/run.py +++ b/scripts/run.py @@ -1,55 +1,36 @@ -from run_tools import FSCT, directory_mode, file_mode -from other_parameters import other_parameters +from scripts.run_tools import FSCT, directory_mode, file_mode +from scripts.tools import parse_config +import configparser -if __name__ == "__main__": - """Choose one of the following or modify as needed. + +def exec(mode, args): + """ + Choose one of the following or modify as needed. Directory mode will find all .las files within a directory and sub directories but will ignore any .las files in folders with "FSCT_output" in their names. - File mode will allow you to select multiple .las files within a directory. - Alternatively, you can just list the point cloud file paths. - If you have multiple point clouds and wish to enter plot coords for each, have a look at "run_with_multiple_plot_centres.py" """ - # point_clouds_to_process = directory_mode() - # point_clouds_to_process = ['full_path_to_your_point_cloud.las', 'full_path_to_your_second_point_cloud.las', etc.] - point_clouds_to_process = file_mode() + if mode == "-d": + point_clouds_to_process = directory_mode() + # file mode + elif mode == "-f": + point_clouds_to_process = args + # attended user file mode + elif mode == "-a": + point_clouds_to_process = file_mode() - for point_cloud_filename in point_clouds_to_process: - parameters = dict( - point_cloud_filename=point_cloud_filename, - # Adjust if needed - plot_centre=None, # [X, Y] Coordinates of the plot centre (metres). If "None", plot_centre is computed based on the point cloud bounding box. - # Circular Plot options - Leave at 0 if not using. - plot_radius=0, # If 0 m, the plot is not cropped. Otherwise, the plot is cylindrically cropped from the plot centre with plot_radius + plot_radius_buffer. - plot_radius_buffer=0, # See README. If non-zero, this is used for "Tree Aware Plot Cropping Mode". - # Set these appropriately for your hardware. - batch_size=2, # You will get CUDA errors if this is too high, as you will run out of VRAM. This won't be an issue if running on CPU only. Must be >= 2. - num_cpu_cores=0, # Number of CPU cores you want to use. If you run out of RAM, lower this. 0 means ALL cores. - use_CPU_only=False, # Set to True if you do not have an Nvidia GPU, or if you don't have enough vRAM. - # Optional settings - Generally leave as they are. - slice_thickness=0.15, # If your point cloud resolution is a bit low (and only if the stem segmentation is still reasonably accurate), try increasing this to 0.2. - # If your point cloud is really dense, you may get away with 0.1. - slice_increment=0.05, # The smaller this is, the better your results will be, however, this increases the run time. - sort_stems=1, # If you don't need the sorted stem points, turning this off speeds things up. - # Veg sorting is required for tree height measurement, but stem sorting isn't necessary for standard use. - height_percentile=100, # If the data contains noise above the canopy, you may wish to set this to the 98th percentile of height, otherwise leave it at 100. - tree_base_cutoff_height=5, # A tree must have a cylinder measurement below this height above the DTM to be kept. This filters unsorted branches from being called individual trees. - generate_output_point_cloud=1, # Turn on if you would like a semantic and instance segmented point cloud. This mode will override the "sort_stems" setting if on. - # If you activate "tree aware plot cropping mode", this function will use it. - ground_veg_cutoff_height=3, # Any vegetation points below this height are considered to be understory and are not assigned to individual trees. - veg_sorting_range=1.5, # Vegetation points can be, at most, this far away from a cylinder horizontally to be matched to a particular tree. - stem_sorting_range=1, # Stem points can be, at most, this far away from a cylinder in 3D to be matched to a particular tree. - taper_measurement_height_min=0, # Lowest height to measure diameter for taper output. - taper_measurement_height_max=30, # Highest height to measure diameter for taper output. - taper_measurement_height_increment=0.2, # diameter measurement increment. - taper_slice_thickness=0.4, # Cylinder measurements within +/- 0.5*taper_slice_thickness are used for taper measurement at a given height. The largest diameter is used. - delete_working_directory=True, # Generally leave this on. Deletes the files used for segmentation after segmentation is finished. - # You may wish to turn it off if you want to re-run/modify the segmentation code so you don't need to run pre-processing every time. - minimise_output_size_mode=0, # Will delete a number of non-essential outputs to reduce storage use. - ) + parser = configparser.ConfigParser() + parser.read("config.ini") + main_parameters = parser['FSCT_main_parameters'] + main_parameters = parse_config(main_parameters) + other_parameters = parser['FSCT_other_parameters'] + other_parameters = parse_config(other_parameters) + for point_cloud_filename in point_clouds_to_process: + parameters = dict(point_cloud_filename=point_cloud_filename) + parameters.update(main_parameters) parameters.update(other_parameters) FSCT( parameters=parameters, diff --git a/scripts/run_tools.py b/scripts/run_tools.py index feee797..7163da8 100644 --- a/scripts/run_tools.py +++ b/scripts/run_tools.py @@ -1,8 +1,8 @@ -from preprocessing import Preprocessing -from inference import SemanticSegmentation -from post_segmentation_script import PostProcessing -from measure import MeasureTree -from report_writer import ReportWriter +from scripts.preprocessing import Preprocessing +from scripts.inference import SemanticSegmentation +from scripts.post_segmentation_script import PostProcessing +from scripts.measure import MeasureTree +from scripts.report_writer import ReportWriter import glob import tkinter as tk import tkinter.filedialog as fd diff --git a/scripts/run_with_multiple_plot_centres.py b/scripts/run_with_multiple_plot_centres.py index c8fc5ed..aa7f8a1 100644 --- a/scripts/run_with_multiple_plot_centres.py +++ b/scripts/run_with_multiple_plot_centres.py @@ -1,58 +1,23 @@ -from run_tools import FSCT, directory_mode, file_mode -from other_parameters import other_parameters +from scripts.run_tools import FSCT +import configparser -if __name__ == "__main__": +def exec(mode, args): """ This script is an example of how to provide multiple different plot centres with your input point clouds. + @args + [[*.las, [X,Y], radius], [...], [...]] """ - point_clouds_to_process = [ - ["E:/your_point_cloud1.las", [your_plot_centre_X_coord, your_plot_centre_Y_coord], your_plot_radius], - ["E:/your_point_cloud2.las", [your_plot_centre_X_coord, your_plot_centre_Y_coord], your_plot_radius], - ] - + point_clouds_to_process = args + config_file = configparser.ConfigParser() + config_file.read("config.ini") + main_parameters = config_file['FSCT_main_parameters'] + other_parameters = config_file['FSCT_other_parameters'] for point_cloud_filename, plot_centre, plot_radius in point_clouds_to_process: - parameters = dict( - point_cloud_filename=point_cloud_filename, - plot_centre=plot_centre, - plot_radius=plot_radius, - plot_radius_buffer=0, - batch_size=18, - num_cpu_cores=0, - use_CPU_only=False, - # Optional settings - Generally leave as they are. - slice_thickness=0.15, - # If your point cloud resolution is a bit low (and only if the stem segmentation is still reasonably accurate), try increasing this to 0.2. - # If your point cloud is really dense, you may get away with 0.1. - slice_increment=0.05, - # The smaller this is, the better your results will be, however, this increases the run time. - sort_stems=1, # If you don't need the sorted stem points, turning this off speeds things up. - # Veg sorting is required for tree height measurement, but stem sorting isn't necessary for standard use. - height_percentile=100, - # If the data contains noise above the canopy, you may wish to set this to the 98th percentile of height, otherwise leave it at 100. - tree_base_cutoff_height=5, - # A tree must have a cylinder measurement below this height above the DTM to be kept. This filters unsorted branches from being called individual trees. - generate_output_point_cloud=1, - # Turn on if you would like a semantic and instance segmented point cloud. This mode will override the "sort_stems" setting if on. - # If you activate "tree aware plot cropping mode", this function will use it. - ground_veg_cutoff_height=3, - # Any vegetation points below this height are considered to be understory and are not assigned to individual trees. - veg_sorting_range=1.5, - # Vegetation points can be, at most, this far away from a cylinder horizontally to be matched to a particular tree. - stem_sorting_range=1, - # Stem points can be, at most, this far away from a cylinder in 3D to be matched to a particular tree. - taper_measurement_height_min=0, # Lowest height to measure diameter for taper output. - taper_measurement_height_max=30, # Highest height to measure diameter for taper output. - taper_measurement_height_increment=0.2, # diameter measurement increment. - taper_slice_thickness=0.4, - # Cylinder measurements within +/- 0.5*taper_slice_thickness are used for taper measurement at a given height. The largest diameter is used. - delete_working_directory=True, - # Generally leave this on. Deletes the files used for segmentation after segmentation is finished. - # You may wish to turn it off if you want to re-run/modify the segmentation code so you don't need to run pre-processing every time. - minimise_output_size_mode=0, - # Will delete a number of non-essential outputs to reduce storage use. - ) - + parameters = dict(point_cloud_filename=point_cloud_filename) + parameters.update(main_parameters) + parameters.update(other_parameters), + parameters.update(plot_centre=plot_centre, plot_radius=plot_radius) parameters.update(other_parameters) FSCT( parameters=parameters, diff --git a/scripts/tools.py b/scripts/tools.py index 2d1f38d..ec30cdc 100644 --- a/scripts/tools.py +++ b/scripts/tools.py @@ -1,9 +1,8 @@ +from configparser import SectionProxy +import json from sklearn.neighbors import NearestNeighbors import numpy as np -import glob import laspy -from sklearn.neighbors import NearestNeighbors -from multiprocessing import Pool, get_context import pandas as pd import os import shutil @@ -292,3 +291,21 @@ def get_taper(single_tree_cyls, slice_heights, tree_base_height, taper_slice_thi else: diameters.append(0) return np.hstack((np.array([[plot_id, tree_id, x_base, y_base, z_base]]), np.array([diameters]))) + + +def parse_config(config: SectionProxy): + list = dict() + for key, value in config.items(): + try: # will handle both ints and floats, even tuples with ints/floats + val = config.get(value) + if value == 'None': + val = None + elif value == 'True' or value == 'False': + val = config.getboolean(key) + else: + val = json.loads(value) + except json.JSONDecodeError: # this means it's a string or a tuple with strings + val = config.get(value) + pass + list[key] = val + return list diff --git a/scripts/train.py b/scripts/train.py index 7c91509..8b1717f 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -1,11 +1,11 @@ -from tools import load_file, save_file, get_fsct_path -from model import Net -from train_datasets import TrainingDataset, ValidationDataset -from fsct_exceptions import NoDataFound +from scripts.fsct_exceptions import NoDataFound import numpy as np import torch import torch.nn as nn import torch.optim as optim +from scripts.model import Net +from scripts.tools import load_file, save_file, get_fsct_path +from train_datasets import TrainingDataset, ValidationDataset from torch_geometric.data import DataLoader import glob import random diff --git a/scripts/train_datasets.py b/scripts/train_datasets.py index af2ef91..da6c757 100644 --- a/scripts/train_datasets.py +++ b/scripts/train_datasets.py @@ -1,5 +1,5 @@ import torch -from torch_geometric.data import Dataset, DataLoader, Data +from torch_geometric.data import Dataset, Data import numpy as np import random import glob diff --git a/scripts/training_monitor.py b/scripts/training_monitor.py index c31c69d..0a1d181 100644 --- a/scripts/training_monitor.py +++ b/scripts/training_monitor.py @@ -1,6 +1,6 @@ import matplotlib.pyplot as plt import numpy as np -from tools import get_fsct_path +from scripts.tools import get_fsct_path def training_plotter():