Cura/plugins/3MFReader/ThreeMFWorkspaceReader.py

# Copyright (c) 2017 Ultimaker B.V.
# Cura is released under the terms of the LGPLv3 or higher.

from configparser import ConfigParser
import zipfile
import os
import threading

import xml.etree.ElementTree as ET

from UM.Workspace.WorkspaceReader import WorkspaceReader
from UM.Application import Application

from UM.Logger import Logger
from UM.i18n import i18nCatalog
from UM.Signal import postponeSignals, CompressTechnique
from UM.Settings.ContainerStack import ContainerStack
from UM.Settings.DefinitionContainer import DefinitionContainer
from UM.Settings.InstanceContainer import InstanceContainer
from UM.Settings.ContainerRegistry import ContainerRegistry
from UM.MimeTypeDatabase import MimeTypeDatabase
from UM.Job import Job
from UM.Preferences import Preferences

from cura.Settings.CuraStackBuilder import CuraStackBuilder
from cura.Settings.ExtruderStack import ExtruderStack
from cura.Settings.GlobalStack import GlobalStack
from cura.Settings.CuraContainerStack import _ContainerIndexes
from cura.CuraApplication import CuraApplication

from .WorkspaceDialog import WorkspaceDialog

i18n_catalog = i18nCatalog("cura")


#
# HACK:
#
# In project loading, when override the existing machine is selected, the stacks and containers that are correctly
# active in the system will be overridden at runtime. Because the project loading is done in a different thread than
# the Qt thread, something else can kick in the middle of the process. One of them is the rendering. It will access
# the current stacks and container, which have not completely been updated yet, so Cura will crash in this case.
#
# This "@call_on_qt_thread" decorator makes sure that a function will always be called on the Qt thread (blocking).
# It is applied to the read() function of project loading so it can be guaranteed that only after the project loading
# process is completely done, everything else that needs to occupy the QT thread will be executed.
#
class InterCallObject:
    def __init__(self):
        self.finish_event = threading.Event()
        self.result = None


def call_on_qt_thread(func):
    def _call_on_qt_thread_wrapper(*args, **kwargs):
        def _handle_call(ico, *args, **kwargs):
            ico.result = func(*args, **kwargs)
            ico.finish_event.set()
        inter_call_object = InterCallObject()
        new_args = tuple([inter_call_object] + list(args)[:])
        CuraApplication.getInstance().callLater(_handle_call, *new_args, **kwargs)
        inter_call_object.finish_event.wait()
        return inter_call_object.result
    return _call_on_qt_thread_wrapper


class ContainerInfo:
    def __init__(self, file_name: str, serialized: str, parser: ConfigParser):
        self.file_name = file_name
        self.serialized = serialized
        self.parser = parser
        self.container = None
        self.definition_id = None


class QualityChangesInfo:
    def __init__(self):
        self.name = None
        self.global_info = None
        self.extruder_info_dict = {}


class MachineInfo:
    def __init__(self):
        self.container_id = None
        self.name = None
        self.definition_id = None
        self.quality_type = None
        self.custom_quality_name = None
        self.quality_changes_info = None
        self.variant_info = None

        self.definition_changes_info = None
        self.user_changes_info = None

        self.extruder_info_dict = {}


class ExtruderInfo:
    def __init__(self):
        self.position = None
        self.variant_info = None
        self.root_material_id = None

        self.definition_changes_info = None
        self.user_changes_info = None


##    Base implementation for reading 3MF workspace files.
class ThreeMFWorkspaceReader(WorkspaceReader):
    def __init__(self):
        super().__init__()
        self._supported_extensions = [".3mf"]
        self._dialog = WorkspaceDialog()
        self._3mf_mesh_reader = None
        self._container_registry = ContainerRegistry.getInstance()

        # suffixes registered with the MineTypes don't start with a dot '.'
        self._definition_container_suffix = "." + ContainerRegistry.getMimeTypeForContainer(DefinitionContainer).preferredSuffix
        self._material_container_suffix = None # We have to wait until all other plugins are loaded before we can set it
        self._instance_container_suffix = "." + ContainerRegistry.getMimeTypeForContainer(InstanceContainer).preferredSuffix
        self._container_stack_suffix = "." + ContainerRegistry.getMimeTypeForContainer(ContainerStack).preferredSuffix
        self._extruder_stack_suffix = "." + ContainerRegistry.getMimeTypeForContainer(ExtruderStack).preferredSuffix
        self._global_stack_suffix = "." + ContainerRegistry.getMimeTypeForContainer(GlobalStack).preferredSuffix

        # Certain instance container types are ignored because we make the assumption that only we make those types
        # of containers. They are:
        #  - quality
        #  - variant
        self._ignored_instance_container_types = {"quality", "variant"}

        self._resolve_strategies = {}

        self._id_mapping = {}

        # In Cura 2.5 and 2.6, the empty profiles used to have those long names
        self._old_empty_profile_id_dict = {"empty_%s" % k: "empty" for k in ["material", "variant"]}

        self._is_same_machine_type = False
        self._old_new_materials = {}
        self._materials_to_select = {}
        self._machine_info = None

    def _clearState(self):
        self._is_same_machine_type = False
        self._id_mapping = {}
        self._old_new_materials = {}
        self._materials_to_select = {}
        self._machine_info = None

    ##  Get a unique name based on the old_id. This is different from directly calling the registry in that it caches results.
    #   This has nothing to do with speed, but with getting consistent new naming for instances & objects.
    def getNewId(self, old_id):
        if old_id not in self._id_mapping:
            self._id_mapping[old_id] = self._container_registry.uniqueName(old_id)
        return self._id_mapping[old_id]

    ##  Separates the given file list into a list of GlobalStack files and a list of ExtruderStack files.
    #
    #   In old versions, extruder stack files have the same suffix as container stack files ".stack.cfg".
    #
    def _determineGlobalAndExtruderStackFiles(self, project_file_name, file_list):
        archive = zipfile.ZipFile(project_file_name, "r")

        global_stack_file_list = [name for name in file_list if name.endswith(self._global_stack_suffix)]
        extruder_stack_file_list = [name for name in file_list if name.endswith(self._extruder_stack_suffix)]

        # separate container stack files and extruder stack files
        files_to_determine = [name for name in file_list if name.endswith(self._container_stack_suffix)]
        for file_name in files_to_determine:
            # FIXME: HACK!
            # We need to know the type of the stack file, but we can only know it if we deserialize it.
            # The default ContainerStack.deserialize() will connect signals, which is not desired in this case.
            # Since we know that the stack files are INI files, so we directly use the ConfigParser to parse them.
            serialized = archive.open(file_name).read().decode("utf-8")
            stack_config = ConfigParser(interpolation = None)
            stack_config.read_string(serialized)

            # sanity check
            if not stack_config.has_option("metadata", "type"):
                Logger.log("e", "%s in %s doesn't seem to be valid stack file", file_name, project_file_name)
                continue

            stack_type = stack_config.get("metadata", "type")
            if stack_type == "extruder_train":
                extruder_stack_file_list.append(file_name)
            elif stack_type == "machine":
                global_stack_file_list.append(file_name)
            else:
                Logger.log("w", "Unknown container stack type '%s' from %s in %s",
                           stack_type, file_name, project_file_name)

        if len(global_stack_file_list) != 1:
            raise RuntimeError("More than one global stack file found: [%s]" % str(global_stack_file_list))

        return global_stack_file_list[0], extruder_stack_file_list

    ##  read some info so we can make decisions
    #   \param file_name
    #   \param show_dialog  In case we use preRead() to check if a file is a valid project file, we don't want to show a dialog.
    def preRead(self, file_name, show_dialog=True, *args, **kwargs):
        self._clearState()

        self._3mf_mesh_reader = Application.getInstance().getMeshFileHandler().getReaderForFile(file_name)
        if self._3mf_mesh_reader and self._3mf_mesh_reader.preRead(file_name) == WorkspaceReader.PreReadResult.accepted:
            pass
        else:
            Logger.log("w", "Could not find reader that was able to read the scene data for 3MF workspace")
            return WorkspaceReader.PreReadResult.failed

        self._machine_info = MachineInfo()
        machine_type = ""
        variant_type_name = i18n_catalog.i18nc("@label", "Nozzle")

        # Check if there are any conflicts, so we can ask the user.
        archive = zipfile.ZipFile(file_name, "r")
        cura_file_names = [name for name in archive.namelist() if name.startswith("Cura/")]

        # A few lists of containers in this project files.
        # When loading the global stack file, it may be associated with those containers, which may or may not be
        # in Cura already, so we need to provide them as alternative search lists.
        instance_container_list = []

        resolve_strategy_keys = ["machine", "material", "quality_changes"]
        self._resolve_strategies = {k: None for k in resolve_strategy_keys}
        containers_found_dict = {k: False for k in resolve_strategy_keys}

        #
        # Read definition containers
        #
        machine_definition_id = None
        machine_definition_container_count = 0
        extruder_definition_container_count = 0
        definition_container_files = [name for name in cura_file_names if name.endswith(self._definition_container_suffix)]
        for definition_container_file in definition_container_files:
            container_id = self._stripFileToId(definition_container_file)
            definitions = self._container_registry.findDefinitionContainersMetadata(id = container_id)
            serialized = archive.open(definition_container_file).read().decode("utf-8")

            if not definitions:
                definition_container = DefinitionContainer.deserializeMetadata(serialized, container_id)[0]
            else:
                definition_container = definitions[0]

            definition_container_type = definition_container.get("type")
            if definition_container_type == "machine":
                machine_definition_id = container_id
                machine_type = definition_container["name"]
                variant_type_name = definition_container.get("variants_name", variant_type_name)

                machine_definition_container_count += 1
            elif definition_container_type == "extruder":
                extruder_definition_container_count += 1
            else:
                Logger.log("w", "Unknown definition container type %s for %s",
                           definition_container_type, definition_container_file)
            Job.yieldThread()

        if machine_definition_container_count != 1:
            return WorkspaceReader.PreReadResult.failed  # Not a workspace file but ordinary 3MF.

        material_labels = []
        material_conflict = False
        xml_material_profile = self._getXmlProfileClass()
        reverse_material_id_dict = {}
        if self._material_container_suffix is None:
            self._material_container_suffix = ContainerRegistry.getMimeTypeForContainer(xml_material_profile).preferredSuffix
        if xml_material_profile:
            material_container_files = [name for name in cura_file_names if name.endswith(self._material_container_suffix)]
            for material_container_file in material_container_files:
                container_id = self._stripFileToId(material_container_file)

                from hashlib import sha1
                hex_container_id = sha1(container_id.encode('utf-8')).hexdigest()

                serialized = archive.open(material_container_file).read().decode("utf-8")
                metadata_list = xml_material_profile.deserializeMetadata(serialized, hex_container_id)
                reverse_map = {metadata["id"].replace(hex_container_id, container_id): container_id.replace(hex_container_id, container_id)
                               for metadata in metadata_list}
                reverse_material_id_dict.update(reverse_map)

                material_labels.append(self._getMaterialLabelFromSerialized(serialized))
                if self._container_registry.findContainersMetadata(id = container_id): #This material already exists.
                    containers_found_dict["material"] = True
                    if not self._container_registry.isReadOnly(container_id):  # Only non readonly materials can be in conflict
                        material_conflict = True
                Job.yieldThread()

        # Check if any quality_changes instance container is in conflict.
        instance_container_files = [name for name in cura_file_names if name.endswith(self._instance_container_suffix)]
        quality_name = ""
        num_settings_overriden_by_quality_changes = 0 # How many settings are changed by the quality changes
        num_user_settings = 0
        quality_changes_conflict = False

        self._machine_info.quality_changes_info = QualityChangesInfo()

        quality_changes_info_list = []
        instance_container_info_dict = {}  # id -> parser
        for instance_container_file_name in instance_container_files:
            container_id = self._stripFileToId(instance_container_file_name)

            serialized = archive.open(instance_container_file_name).read().decode("utf-8")
            serialized = InstanceContainer._updateSerialized(serialized, instance_container_file_name)
            parser = ConfigParser(interpolation = None)
            parser.read_string(serialized)
            container_info = ContainerInfo(instance_container_file_name, serialized, parser)
            instance_container_info_dict[container_id] = container_info

            instance_container = InstanceContainer(container_id)

            # Deserialize InstanceContainer by converting read data from bytes to string
            instance_container.deserialize(serialized, file_name = instance_container_file_name)
            instance_container_list.append(instance_container)

            container_type = instance_container.getMetaDataEntry("type")
            if container_type == "quality_changes":
                quality_changes_info_list.append(container_info)

                if not parser.has_option("metadata", "extruder"):
                    self._machine_info.quality_changes_info.name = parser["general"]["name"]
                    self._machine_info.quality_changes_info.global_info = container_info

                quality_name = instance_container.getName()
                num_settings_overriden_by_quality_changes += len(instance_container._instances)
                # Check if quality changes already exists.
                quality_changes = self._container_registry.findInstanceContainers(id = container_id)
                if quality_changes:
                    containers_found_dict["quality_changes"] = True
                    # Check if there really is a conflict by comparing the values
                    if quality_changes[0] != instance_container:
                        quality_changes_conflict = True
            elif container_type == "quality":
                if not quality_name:
                    quality_name = instance_container.getName()
            elif container_type == "user":
                num_user_settings += len(instance_container._instances)
            elif container_type in self._ignored_instance_container_types:
                # Ignore certain instance container types
                Logger.log("w", "Ignoring instance container [%s] with type [%s]", container_id, container_type)
                continue

            Job.yieldThread()

        if self._machine_info.quality_changes_info.global_info is None:
            self._machine_info.quality_changes_info = None

        # Load ContainerStack files and ExtruderStack files
        global_stack_file, extruder_stack_files = self._determineGlobalAndExtruderStackFiles(
            file_name, cura_file_names)
        machine_conflict = False
        # Because there can be cases as follows:
        #  - the global stack exists but some/all of the extruder stacks DON'T exist
        #  - the global stack DOESN'T exist but some/all of the extruder stacks exist
        # To simplify this, only check if the global stack exists or not
        global_stack_id = self._stripFileToId(global_stack_file)
        serialized = archive.open(global_stack_file).read().decode("utf-8")
        machine_name = self._getMachineNameFromSerializedStack(serialized)
        stacks = self._container_registry.findContainerStacks(name = machine_name, type = "machine")
        self._is_same_machine_type = True
        if stacks:
            global_stack = stacks[0]
            containers_found_dict["machine"] = True
            # Check if there are any changes at all in any of the container stacks.
            id_list = self._getContainerIdListFromSerialized(serialized)
            for index, container_id in enumerate(id_list):
                # take into account the old empty container IDs
                container_id = self._old_empty_profile_id_dict.get(container_id, container_id)
                if global_stack.getContainer(index).getId() != container_id:
                    machine_conflict = True
                    break
            self._is_same_machine_type = global_stack.definition.getId() == machine_definition_id

        # Get quality type
        parser = ConfigParser(interpolation = None)
        parser.read_string(serialized)
        quality_container_id = parser["containers"][str(_ContainerIndexes.Quality)]
        quality_type = instance_container_info_dict[quality_container_id].parser["metadata"]["quality_type"]

        # Get machine info
        serialized = archive.open(global_stack_file).read().decode("utf-8")
        serialized = GlobalStack._updateSerialized(serialized, global_stack_file)
        parser = ConfigParser(interpolation = None)
        parser.read_string(serialized)
        definition_changes_id = parser["containers"][str(_ContainerIndexes.DefinitionChanges)]
        if definition_changes_id not in ("empty", "empty_definition_changes"):
            self._machine_info.definition_changes_info = instance_container_info_dict[definition_changes_id]
        user_changes_id = parser["containers"][str(_ContainerIndexes.UserChanges)]
        if user_changes_id not in ("empty", "empty_user_changes"):
            self._machine_info.user_changes_info = instance_container_info_dict[user_changes_id]

        # Also check variant and material in case it doesn't have extruder stacks
        if not extruder_stack_files:
            position = "0"

            extruder_info = ExtruderInfo()
            extruder_info.position = position
            variant_id = parser["containers"][str(_ContainerIndexes.Variant)]
            material_id = parser["containers"][str(_ContainerIndexes.Material)]
            if variant_id not in ("empty", "empty_variant"):
                extruder_info.variant_info = instance_container_info_dict[variant_id]
            if material_id not in ("empty", "empty_material"):
                root_material_id = reverse_material_id_dict[material_id]
                extruder_info.root_material_id = root_material_id
            self._machine_info.extruder_info_dict[position] = extruder_info
        else:
            variant_id = parser["containers"][str(_ContainerIndexes.Variant)]
            if variant_id not in ("empty", "empty_variant"):
                self._machine_info.variant_info = instance_container_info_dict[variant_id]

        Job.yieldThread()

        # if the global stack is found, we check if there are conflicts in the extruder stacks
        for extruder_stack_file in extruder_stack_files:
            serialized = archive.open(extruder_stack_file).read().decode("utf-8")
            serialized = ExtruderStack._updateSerialized(serialized, extruder_stack_file)
            parser = ConfigParser(interpolation = None)
            parser.read_string(serialized)

            # The check should be done for the extruder stack that's associated with the existing global stack,
            # and those extruder stacks may have different IDs.
            # So we check according to the positions
            position = parser["metadata"]["position"]
            variant_id = parser["containers"][str(_ContainerIndexes.Variant)]
            material_id = parser["containers"][str(_ContainerIndexes.Material)]

            extruder_info = ExtruderInfo()
            extruder_info.position = position
            if variant_id not in ("empty", "empty_variant"):
                extruder_info.variant_info = instance_container_info_dict[variant_id]
            if material_id not in ("empty", "empty_material"):
                root_material_id = reverse_material_id_dict[material_id]
                extruder_info.root_material_id = root_material_id
            definition_changes_id = parser["containers"][str(_ContainerIndexes.DefinitionChanges)]
            if definition_changes_id not in ("empty", "empty_definition_changes"):
                extruder_info.definition_changes_info = instance_container_info_dict[definition_changes_id]
            user_changes_id = parser["containers"][str(_ContainerIndexes.UserChanges)]
            if user_changes_id not in ("empty", "empty_user_changes"):
                extruder_info.user_changes_info = instance_container_info_dict[user_changes_id]
            self._machine_info.extruder_info_dict[position] = extruder_info

            if not machine_conflict and containers_found_dict["machine"]:
                if position not in global_stack.extruders:
                    continue

                existing_extruder_stack = global_stack.extruders[position]
                # check if there are any changes at all in any of the container stacks.
                id_list = self._getContainerIdListFromSerialized(serialized)
                for index, container_id in enumerate(id_list):
                    # take into account the old empty container IDs
                    container_id = self._old_empty_profile_id_dict.get(container_id, container_id)
                    if existing_extruder_stack.getContainer(index).getId() != container_id:
                        machine_conflict = True
                        break

        if self._machine_info.quality_changes_info is not None:
            for quality_changes_info in quality_changes_info_list:
                if not quality_changes_info.parser.has_option("metadata", "extruder"):
                    continue
                extruder_definition_id = quality_changes_info.parser["metadata"]["extruder"]
                extruder_definition_metadata = self._container_registry.findDefinitionContainersMetadata(id = extruder_definition_id)[0]
                position = extruder_definition_metadata["position"]
                self._machine_info.quality_changes_info.extruder_info_dict[position] = quality_changes_info

        num_visible_settings = 0
        try:
            temp_preferences = Preferences()
            serialized = archive.open("Cura/preferences.cfg").read().decode("utf-8")
            temp_preferences.deserialize(serialized)

            visible_settings_string = temp_preferences.getValue("general/visible_settings")
            has_visible_settings_string = visible_settings_string is not None
            if visible_settings_string is not None:
                num_visible_settings = len(visible_settings_string.split(";"))
            active_mode = temp_preferences.getValue("cura/active_mode")
            if not active_mode:
                active_mode = Preferences.getInstance().getValue("cura/active_mode")
        except KeyError:
            # If there is no preferences file, it's not a workspace, so notify user of failure.
            Logger.log("w", "File %s is not a valid workspace.", file_name)
            return WorkspaceReader.PreReadResult.failed

        # In case we use preRead() to check if a file is a valid project file, we don't want to show a dialog.
        if not show_dialog:
            return WorkspaceReader.PreReadResult.accepted

        # prepare data for the dialog
        num_extruders = extruder_definition_container_count
        if num_extruders == 0:
            num_extruders = 1  # No extruder stacks found, which means there is one extruder

        extruders = num_extruders * [""]

        self._machine_info.container_id = global_stack_id
        self._machine_info.name = machine_name
        self._machine_info.definition_id = machine_definition_id
        self._machine_info.quality_type = quality_type
        self._machine_info.custom_quality_name = quality_name

        if machine_conflict and not self._is_same_machine_type:
            machine_conflict = False

        # Show the dialog, informing the user what is about to happen.
        self._dialog.setMachineConflict(machine_conflict)
        self._dialog.setQualityChangesConflict(quality_changes_conflict)
        self._dialog.setMaterialConflict(material_conflict)
        self._dialog.setHasVisibleSettingsField(has_visible_settings_string)
        self._dialog.setNumVisibleSettings(num_visible_settings)
        self._dialog.setQualityName(quality_name)
        self._dialog.setQualityType(quality_type)
        self._dialog.setNumSettingsOverridenByQualityChanges(num_settings_overriden_by_quality_changes)
        self._dialog.setNumUserSettings(num_user_settings)
        self._dialog.setActiveMode(active_mode)
        self._dialog.setMachineName(machine_name)
        self._dialog.setMaterialLabels(material_labels)
        self._dialog.setMachineType(machine_type)
        self._dialog.setExtruders(extruders)
        self._dialog.setVariantType(variant_type_name)
        self._dialog.setHasObjectsOnPlate(Application.getInstance().platformActivity)
        self._dialog.show()

        # Block until the dialog is closed.
        self._dialog.waitForClose()

        if self._dialog.getResult() == {}:
            return WorkspaceReader.PreReadResult.cancelled

        self._resolve_strategies = self._dialog.getResult()
        #
        # There can be 3 resolve strategies coming from the dialog:
        #  - new:       create a new container
        #  - override:  override the existing container
        #  - None:      There is no conflict, which means containers with the same IDs may or may not be there already.
        #               If there is an existing container, there is no conflict between them, and default to "override"
        #               If there is no existing container, default to "new"
        #
        # Default values
        for key, strategy in self._resolve_strategies.items():
            if key not in containers_found_dict or strategy is not None:
                continue
            self._resolve_strategies[key] = "override" if containers_found_dict[key] else "new"

        return WorkspaceReader.PreReadResult.accepted

    ## Overrides an ExtruderStack in the given GlobalStack and returns the new ExtruderStack.
    def _overrideExtruderStack(self, global_stack, extruder_file_content, extruder_stack_file):
        # Get extruder position first
        extruder_config = ConfigParser(interpolation = None)
        extruder_config.read_string(extruder_file_content)
        if not extruder_config.has_option("metadata", "position"):
            msg = "Could not find 'metadata/position' in extruder stack file"
            Logger.log("e", "Could not find 'metadata/position' in extruder stack file")
            raise RuntimeError(msg)
        extruder_position = extruder_config.get("metadata", "position")
        try:
            extruder_stack = global_stack.extruders[extruder_position]
        except KeyError:
            Logger.log("w", "Could not find the matching extruder stack to override for position %s", extruder_position)
            return None

        # Override the given extruder stack
        extruder_stack.deserialize(extruder_file_content, file_name = extruder_stack_file)

        # return the new ExtruderStack
        return extruder_stack

    ##  Read the project file
    #   Add all the definitions / materials / quality changes that do not exist yet. Then it loads
    #   all the stacks into the container registry. In some cases it will reuse the container for the global stack.
    #   It handles old style project files containing .stack.cfg as well as new style project files
    #   containing global.cfg / extruder.cfg
    #
    #   \param file_name
    @call_on_qt_thread
    def read(self, file_name):
        container_registry = ContainerRegistry.getInstance()
        signals = [container_registry.containerAdded,
                   container_registry.containerRemoved,
                   container_registry.containerMetaDataChanged]
        #
        # We now have different managers updating their lookup tables upon container changes. It is critical to make
        # sure that the managers have a complete set of data when they update.
        #
        # In project loading, lots of the container-related signals are loosely emitted, which can create timing gaps
        # for incomplete data update or other kinds of issues to happen.
        #
        # To avoid this, we postpone all signals so they don't get emitted immediately. But, please also be aware that,
        # because of this, do not expect to have the latest data in the lookup tables in project loading.
        #
        with postponeSignals(*signals, compress = CompressTechnique.NoCompression):
            return self._read(file_name)

    def _read(self, file_name):
        application = CuraApplication.getInstance()
        material_manager = application.getMaterialManager()

        archive = zipfile.ZipFile(file_name, "r")

        cura_file_names = [name for name in archive.namelist() if name.startswith("Cura/")]

        # Create a shadow copy of the preferences (we don't want all of the preferences, but we do want to re-use its
        # parsing code.
        temp_preferences = Preferences()
        serialized = archive.open("Cura/preferences.cfg").read().decode("utf-8")
        temp_preferences.deserialize(serialized)

        # Copy a number of settings from the temp preferences to the global
        global_preferences = Preferences.getInstance()

        visible_settings = temp_preferences.getValue("general/visible_settings")
        if visible_settings is None:
            Logger.log("w", "Workspace did not contain visible settings. Leaving visibility unchanged")
        else:
            global_preferences.setValue("general/visible_settings", visible_settings)
            global_preferences.setValue("general/preset_setting_visibility_choice", "Custom")

        categories_expanded = temp_preferences.getValue("cura/categories_expanded")
        if categories_expanded is None:
            Logger.log("w", "Workspace did not contain expanded categories. Leaving them unchanged")
        else:
            global_preferences.setValue("cura/categories_expanded", categories_expanded)

        application.expandedCategoriesChanged.emit()  # Notify the GUI of the change

        # If a machine with the same name is of a different type, always create a new one.
        if not self._is_same_machine_type or self._resolve_strategies["machine"] != "override":
            # We need to create a new machine
            machine_name = self._container_registry.uniqueName(self._machine_info.name)

            global_stack = CuraStackBuilder.createMachine(machine_name, self._machine_info.definition_id)
            extruder_stack_dict = global_stack.extruders

            self._container_registry.addContainer(global_stack)
        else:
            # Find the machine
            global_stack = self._container_registry.findContainerStacks(name = self._machine_info.name, type = "machine")[0]
            extruder_stacks = self._container_registry.findContainerStacks(machine = global_stack.getId(),
                                                                           type = "extruder_train")
            extruder_stack_dict = {stack.getMetaDataEntry("position"): stack for stack in extruder_stacks}

        Logger.log("d", "Workspace loading is checking definitions...")
        # Get all the definition files & check if they exist. If not, add them.
        definition_container_files = [name for name in cura_file_names if name.endswith(self._definition_container_suffix)]
        for definition_container_file in definition_container_files:
            container_id = self._stripFileToId(definition_container_file)
            definitions = self._container_registry.findDefinitionContainersMetadata(id = container_id)
            if not definitions:
                definition_container = DefinitionContainer(container_id)
                definition_container.deserialize(archive.open(definition_container_file).read().decode("utf-8"),
                                                 file_name = definition_container_file)
                self._container_registry.addContainer(definition_container)
            Job.yieldThread()

        Logger.log("d", "Workspace loading is checking materials...")
        # Get all the material files and check if they exist. If not, add them.
        xml_material_profile = self._getXmlProfileClass()
        if self._material_container_suffix is None:
            self._material_container_suffix = ContainerRegistry.getMimeTypeForContainer(xml_material_profile).suffixes[0]
        if xml_material_profile:
            material_container_files = [name for name in cura_file_names if name.endswith(self._material_container_suffix)]
            for material_container_file in material_container_files:
                to_deserialize_material = False
                container_id = self._stripFileToId(material_container_file)
                need_new_name = False
                materials = self._container_registry.findInstanceContainers(id = container_id)

                if not materials:
                    # No material found, deserialize this material later and add it
                    to_deserialize_material = True
                else:
                    material_container = materials[0]
                    old_material_root_id = material_container.getMetaDataEntry("base_file")
                    if not self._container_registry.isReadOnly(old_material_root_id):  # Only create new materials if they are not read only.
                        to_deserialize_material = True

                        if self._resolve_strategies["material"] == "override":
                            # Remove the old materials and then deserialize the one from the project
                            root_material_id = material_container.getMetaDataEntry("base_file")
                            material_manager.removeMaterialByRootId(root_material_id)
                        elif self._resolve_strategies["material"] == "new":
                            # Note that we *must* deserialize it with a new ID, as multiple containers will be
                            # auto created & added.
                            container_id = self.getNewId(container_id)
                            self._old_new_materials[old_material_root_id] = container_id
                            need_new_name = True

                if to_deserialize_material:
                    material_container = xml_material_profile(container_id)
                    material_container.deserialize(archive.open(material_container_file).read().decode("utf-8"),
                                                   file_name = container_id + "." + self._material_container_suffix)
                    if need_new_name:
                        new_name = ContainerRegistry.getInstance().uniqueName(material_container.getName())
                        material_container.setName(new_name)
                    self._container_registry.addContainer(material_container)
                Job.yieldThread()

        # Handle quality changes if any
        self._processQualityChanges(global_stack)

        # Prepare the machine
        self._applyChangesToMachine(global_stack, extruder_stack_dict)

        Logger.log("d", "Workspace loading is notifying rest of the code of changes...")
        # Actually change the active machine.
        #
        # This is scheduled for later is because it depends on the Variant/Material/Qualitiy Managers to have the latest
        # data, but those managers will only update upon a container/container metadata changed signal. Because this
        # function is running on the main thread (Qt thread), although those "changed" signals have been emitted, but
        # they won't take effect until this function is done.
        # To solve this, we schedule _updateActiveMachine() for later so it will have the latest data.
        Application.getInstance().setGlobalContainerStack(global_stack)
        self._updateActiveMachine(global_stack)

        # Load all the nodes / meshdata of the workspace
        nodes = self._3mf_mesh_reader.read(file_name)
        if nodes is None:
            nodes = []

        base_file_name = os.path.basename(file_name)
        if base_file_name.endswith(".curaproject.3mf"):
            base_file_name = base_file_name[:base_file_name.rfind(".curaproject.3mf")]
        self.setWorkspaceName(base_file_name)
        return nodes

    def _processQualityChanges(self, global_stack):
        if self._machine_info.quality_changes_info is None:
            return

        application = CuraApplication.getInstance()
        quality_manager = application.getQualityManager()

        # If we have custom profiles, load them
        quality_changes_name = self._machine_info.quality_changes_info.name
        if self._machine_info.quality_changes_info is not None:
            Logger.log("i", "Loading custom profile [%s] from project file",
                       self._machine_info.quality_changes_info.name)

            # Get the correct extruder definition IDs for quality changes
            from cura.Machines.QualityManager import getMachineDefinitionIDForQualitySearch
            machine_definition_id_for_quality = getMachineDefinitionIDForQualitySearch(global_stack)
            machine_definition_for_quality = self._container_registry.findDefinitionContainers(id = machine_definition_id_for_quality)[0]
            extruder_dict_for_quality = machine_definition_for_quality.getMetaDataEntry("machine_extruder_trains")

            quality_changes_info = self._machine_info.quality_changes_info
            quality_changes_quality_type = quality_changes_info.global_info.parser["metadata"]["quality_type"]

            quality_changes_name = quality_changes_info.name
            create_new = self._resolve_strategies.get("quality_changes") != "override"
            if create_new:
                container_info_dict = {None: self._machine_info.quality_changes_info.global_info}
                container_info_dict.update(quality_changes_info.extruder_info_dict)

                quality_changes_name = self._container_registry.uniqueName(quality_changes_name)
                for position, container_info in container_info_dict.items():
                    extruder_definition_id = None
                    if position is not None:
                        extruder_definition_id = extruder_dict_for_quality[position]

                    container = quality_manager._createQualityChanges(quality_changes_quality_type,
                                                                      quality_changes_name,
                                                                      global_stack, extruder_definition_id)
                    container_info.container = container
                    container.setDirty(True)
                    self._container_registry.addContainer(container)

                    Logger.log("d", "Created new quality changes container [%s]", container.getId())

            else:
                # Find the existing containers
                quality_changes_containers = self._container_registry.findInstanceContainers(name = quality_changes_name,
                                                                                             type = "quality_changes")
                for container in quality_changes_containers:
                    extruder_definition_id = container.getMetaDataEntry("extruder")
                    if not extruder_definition_id:
                        quality_changes_info.global_info.container = container
                    else:
                        extruder_definition_metadata = self._container_registry.findDefinitionContainersMetadata(id = extruder_definition_id)[0]
                        position = extruder_definition_metadata["position"]
                        if position not in quality_changes_info.extruder_info_dict:
                            quality_changes_info.extruder_info_dict[position] = ContainerInfo(None, None, None)
                        container_info = quality_changes_info.extruder_info_dict[position]
                        container_info.container = container

            for position, container_info in quality_changes_info.extruder_info_dict.items():
                container_info.definition_id = extruder_dict_for_quality[position]

            # If there is no quality changes for any extruder, create one.
            if not quality_changes_info.extruder_info_dict:
                container_info = ContainerInfo(None, None, None)
                quality_changes_info.extruder_info_dict["0"] = container_info
                extruder_definition_id = extruder_dict_for_quality["0"]
                container_info.definition_id = extruder_definition_id

                container = quality_manager._createQualityChanges(quality_changes_quality_type, quality_changes_name,
                                                                  global_stack, extruder_definition_id)
                container_info.container = container
                container.setDirty(True)
                self._container_registry.addContainer(container)

                Logger.log("d", "Created new quality changes container [%s]", container.getId())

            # Clear all existing containers
            quality_changes_info.global_info.container.clear()
            for container_info in quality_changes_info.extruder_info_dict.values():
                container_info.container.clear()

            # Loop over everything and override the existing containers
            global_info = quality_changes_info.global_info
            global_info.container.clear()  # Clear all
            for key, value in global_info.parser["values"].items():
                if not machine_definition_for_quality.getProperty(key, "settable_per_extruder"):
                    global_info.container.setProperty(key, "value", value)
                else:
                    quality_changes_info.extruder_info_dict["0"].container.setProperty(key, "value", value)

            for position, container_info in quality_changes_info.extruder_info_dict.items():
                if container_info.parser is None:
                    continue

                if container_info.container is None:
                    extruder_definition_id = extruder_dict_for_quality[position]
                    container = quality_manager._createQualityChanges(quality_changes_quality_type, quality_changes_name,
                                                                      global_stack, extruder_definition_id)
                    container_info.container = container

                for key, value in container_info.parser["values"].items():
                    container_info.container.setProperty(key, "value", value)

        self._machine_info.quality_changes_info.name = quality_changes_name

    def _clearStack(self, stack):
        application = CuraApplication.getInstance()

        stack.definitionChanges.clear()
        stack.variant = application.empty_variant_container
        stack.material = application.empty_material_container
        stack.quality = application.empty_quality_container
        stack.qualityChanges = application.empty_quality_changes_container
        stack.userChanges.clear()

    def _applyDefinitionChanges(self, global_stack, extruder_stack_dict):
        values_to_set_for_extruders = {}
        if self._machine_info.definition_changes_info is not None:
            parser = self._machine_info.definition_changes_info.parser
            for key, value in parser["values"].items():
                if global_stack.getProperty(key, "settable_per_extruder"):
                    values_to_set_for_extruders[key] = value
                else:
                    global_stack.definitionChanges.setProperty(key, "value", value)

        for position, extruder_stack in extruder_stack_dict.items():
            if position not in self._machine_info.extruder_info_dict:
                continue

            extruder_info = self._machine_info.extruder_info_dict[position]
            if extruder_info.definition_changes_info is None:
                continue
            parser = extruder_info.definition_changes_info.parser
            for key, value in values_to_set_for_extruders.items():
                extruder_stack.definitionChanges.setProperty(key, "value", value)
            if parser is not None:
                for key, value in parser["values"].items():
                    extruder_stack.definitionChanges.setProperty(key, "value", value)

    def _applyUserChanges(self, global_stack, extruder_stack_dict):
        values_to_set_for_extruder_0 = {}
        if self._machine_info.user_changes_info is not None:
            parser = self._machine_info.user_changes_info.parser
            for key, value in parser["values"].items():
                if global_stack.getProperty(key, "settable_per_extruder"):
                    values_to_set_for_extruder_0[key] = value
                else:
                    global_stack.userChanges.setProperty(key, "value", value)

        for position, extruder_stack in extruder_stack_dict.items():
            if position not in self._machine_info.extruder_info_dict:
                continue

            extruder_info = self._machine_info.extruder_info_dict[position]
            if extruder_info.user_changes_info is not None:
                parser = self._machine_info.extruder_info_dict[position].user_changes_info.parser
                if position == "0":
                    for key, value in values_to_set_for_extruder_0.items():
                        extruder_stack.userChanges.setProperty(key, "value", value)
                if parser is not None:
                    for key, value in parser["values"].items():
                        extruder_stack.userChanges.setProperty(key, "value", value)

    def _applyVariants(self, global_stack, extruder_stack_dict):
        application = CuraApplication.getInstance()
        variant_manager = application.getVariantManager()

        if self._machine_info.variant_info is not None:
            parser = self._machine_info.variant_info.parser
            variant_name = parser["general"]["name"]

            from cura.Machines.VariantManager import VariantType
            variant_type = VariantType.BUILD_PLATE

            node = variant_manager.getVariantNode(global_stack.definition.getId(), variant_name, variant_type)
            if node is not None:
                global_stack.variant = node.getContainer()

        for position, extruder_stack in extruder_stack_dict.items():
            if position not in self._machine_info.extruder_info_dict:
                continue
            extruder_info = self._machine_info.extruder_info_dict[position]
            if extruder_info.variant_info is None:
                continue
            parser = extruder_info.variant_info.parser

            variant_name = parser["general"]["name"]
            from cura.Machines.VariantManager import VariantType
            variant_type = VariantType.NOZZLE

            node = variant_manager.getVariantNode(global_stack.definition.getId(), variant_name, variant_type)
            if node is not None:
                extruder_stack.variant = node.getContainer()

    def _applyMaterials(self, global_stack, extruder_stack_dict):
        application = CuraApplication.getInstance()
        material_manager = application.getMaterialManager()

        # Force update lookup tables first
        material_manager.initialize()

        for position, extruder_stack in extruder_stack_dict.items():
            if position not in self._machine_info.extruder_info_dict:
                continue
            extruder_info = self._machine_info.extruder_info_dict[position]
            if extruder_info.root_material_id is None:
                continue

            root_material_id = extruder_info.root_material_id
            root_material_id = self._old_new_materials.get(root_material_id, root_material_id)

            # get material diameter of this extruder
            machine_material_diameter = extruder_stack.materialDiameter
            material_node = material_manager.getMaterialNode(global_stack.definition.getId(),
                                                             extruder_stack.variant.getName(),
                                                             machine_material_diameter,
                                                             root_material_id)
            if material_node is not None:
                extruder_stack.material = material_node.getContainer()

    def _applyChangesToMachine(self, global_stack, extruder_stack_dict):
        # Clear all first
        self._clearStack(global_stack)
        for extruder_stack in extruder_stack_dict.values():
            self._clearStack(extruder_stack)

        self._applyDefinitionChanges(global_stack, extruder_stack_dict)
        self._applyUserChanges(global_stack, extruder_stack_dict)
        self._applyVariants(global_stack, extruder_stack_dict)
        self._applyMaterials(global_stack, extruder_stack_dict)

        # prepare the quality to select
        self._quality_changes_to_apply = None
        self._quality_type_to_apply = None
        if self._machine_info.quality_changes_info is not None:
            self._quality_changes_to_apply = self._machine_info.quality_changes_info.name
        else:
            self._quality_type_to_apply = self._machine_info.quality_type

    def _updateActiveMachine(self, global_stack):
        # Actually change the active machine.
        machine_manager = Application.getInstance().getMachineManager()
        material_manager = Application.getInstance().getMaterialManager()
        quality_manager = Application.getInstance().getQualityManager()

        # Force update the lookup maps first
        material_manager.initialize()
        quality_manager.initialize()

        machine_manager.setActiveMachine(global_stack.getId())

        if self._quality_changes_to_apply:
            quality_changes_group_dict = quality_manager.getQualityChangesGroups(global_stack)
            if self._quality_changes_to_apply not in quality_changes_group_dict:
                Logger.log("e", "Could not find quality_changes [%s]", self._quality_changes_to_apply)
                return
            quality_changes_group = quality_changes_group_dict[self._quality_changes_to_apply]
            machine_manager.setQualityChangesGroup(quality_changes_group, no_dialog = True)
        else:
            self._quality_type_to_apply = self._quality_type_to_apply.lower()
            quality_group_dict = quality_manager.getQualityGroups(global_stack)
            if self._quality_type_to_apply in quality_group_dict:
                quality_group = quality_group_dict[self._quality_type_to_apply]
            else:
                Logger.log("i", "Could not find quality type [%s], switch to default", self._quality_type_to_apply)
                preferred_quality_type = global_stack.getMetaDataEntry("preferred_quality_type")
                quality_group_dict = quality_manager.getQualityGroups(global_stack)
                quality_group = quality_group_dict.get(preferred_quality_type)
                if quality_group is None:
                    Logger.log("e", "Could not get preferred quality type [%s]", preferred_quality_type)

            if quality_group is not None:
                machine_manager.setQualityGroup(quality_group, no_dialog = True)

        # Notify everything/one that is to notify about changes.
        global_stack.containersChanged.emit(global_stack.getTop())

    def _stripFileToId(self, file):
        mime_type = MimeTypeDatabase.getMimeTypeForFile(file)
        file = mime_type.stripExtension(file)
        return file.replace("Cura/", "")

    def _getXmlProfileClass(self):
        return self._container_registry.getContainerForMimeType(MimeTypeDatabase.getMimeType("application/x-ultimaker-material-profile"))

    ##  Get the list of ID's of all containers in a container stack by partially parsing it's serialized data.
    def _getContainerIdListFromSerialized(self, serialized):
        parser = ConfigParser(interpolation=None, empty_lines_in_values=False)
        parser.read_string(serialized)

        container_ids = []
        if "containers" in parser:
            for index, container_id in parser.items("containers"):
                container_ids.append(container_id)
        elif parser.has_option("general", "containers"):
            container_string = parser["general"].get("containers", "")
            container_list = container_string.split(",")
            container_ids = [container_id for container_id in container_list if container_id != ""]

        # HACK: there used to be 6 containers numbering from 0 to 5 in a stack,
        #       now we have 7: index 5 becomes "definition_changes"
        if len(container_ids) == 6:
            # Hack; We used to not save the definition changes. Fix this.
            container_ids.insert(5, "empty")

        return container_ids

    def _getMachineNameFromSerializedStack(self, serialized):
        parser = ConfigParser(interpolation=None, empty_lines_in_values=False)
        parser.read_string(serialized)
        return parser["general"].get("name", "")

    def _getMaterialLabelFromSerialized(self, serialized):
        data = ET.fromstring(serialized)
        metadata = data.iterfind("./um:metadata/um:name/um:label", {"um": "http://www.ultimaker.com/material"})
        for entry in metadata:
            return entry.text