Updated HACS and also fixed Garadget #727

config/custom_components/hacs/hacsbase/__init__.py

@@ -9,13 +9,20 @@ from homeassistant.helpers.event import async_call_later, async_track_time_inter
 from aiogithubapi import AIOGitHubException, AIOGitHubRatelimit
 from integrationhelper import Logger
 
-from .task_factory import HacsTaskFactory
-from .exceptions import HacsException
+from custom_components.hacs.hacsbase.task_factory import HacsTaskFactory
+from custom_components.hacs.hacsbase.exceptions import HacsException
 
-from ..const import ELEMENT_TYPES
-from ..setup import setup_extra_stores
-from ..store import async_load_from_store, async_save_to_store
-from ..helpers.get_defaults import get_default_repos_lists, get_default_repos_orgs
+from custom_components.hacs.const import ELEMENT_TYPES
+from custom_components.hacs.setup import setup_extra_stores
+from custom_components.hacs.store import async_load_from_store, async_save_to_store
+from custom_components.hacs.helpers.get_defaults import (
+    get_default_repos_lists,
+    get_default_repos_orgs,
+)
+
+from custom_components.hacs.helpers.register_repository import register_repository
+from custom_components.hacs.globals import removed_repositories, get_removed, is_removed
+from custom_components.hacs.repositories.removed import RemovedRepository
 
 
 class HacsStatus:
@@ -40,7 +47,6 @@ class HacsCommon:
     """Common for HACS."""
 
     categories = []
-    blacklist = []
     default = []
     installed = []
     skip = []
@@ -90,6 +96,7 @@ class Hacs:
     github = None
     hass = None
     version = None
+    session = None
     factory = HacsTaskFactory()
     system = System()
     recuring_tasks = []
@@ -114,7 +121,7 @@ class Hacs:
         """Get repository by full_name."""
         try:
             for repository in self.repositories:
-                if repository.information.full_name == repository_full_name:
+                if repository.data.full_name.lower() == repository_full_name.lower():
                     return repository
         except Exception:  # pylint: disable=broad-except
             pass
@@ -122,10 +129,9 @@ class Hacs:
 
     def is_known(self, repository_full_name):
         """Return a bool if the repository is known."""
-        for repository in self.repositories:
-            if repository.information.full_name == repository_full_name:
-                return True
-        return False
+        return repository_full_name.lower() in [
+            x.data.full_name.lower() for x in self.repositories
+        ]
 
     @property
     def sorted_by_name(self):
@@ -135,59 +141,16 @@ class Hacs:
     @property
     def sorted_by_repository_name(self):
         """Return a sorted(by repository_name) list of repository objects."""
-        return sorted(self.repositories, key=lambda x: x.information.full_name)
+        return sorted(self.repositories, key=lambda x: x.data.full_name)
 
     async def register_repository(self, full_name, category, check=True):
         """Register a repository."""
-        from ..repositories.repository import RERPOSITORY_CLASSES
-
-        if full_name in self.common.skip:
-            if full_name != "hacs/integration":
-                self.logger.debug(f"Skipping {full_name}")
-                return
-
-        if category not in RERPOSITORY_CLASSES:
-            msg = f"{category} is not a valid repository category."
-            self.logger.error(msg)
-            raise HacsException(msg)
-
-        repository = RERPOSITORY_CLASSES[category](full_name)
-        if check:
-            try:
-                await repository.registration()
-                if self.system.status.new:
-                    repository.status.new = False
-                if repository.validate.errors:
-                    self.common.skip.append(repository.information.full_name)
-                    if not self.system.status.startup:
-                        self.logger.error(f"Validation for {full_name} failed.")
-                    return repository.validate.errors
-                repository.logger.info("Registration complete")
-            except AIOGitHubException as exception:
-                self.logger.debug(self.github.ratelimits.remaining)
-                self.logger.debug(self.github.ratelimits.reset_utc)
-                self.common.skip.append(repository.information.full_name)
-                # if not self.system.status.startup:
-                if self.system.status.startup:
-                    self.logger.error(
-                        f"Validation for {full_name} failed with {exception}."
-                    )
-                return exception
-        self.hass.bus.async_fire(
-            "hacs/repository",
-            {
-                "id": 1337,
-                "action": "registration",
-                "repository": repository.information.full_name,
-                "repository_id": repository.information.uid,
-            },
-        )
-        self.repositories.append(repository)
+        await register_repository(full_name, category, check=True)
 
     async def startup_tasks(self):
         """Tasks tha are started after startup."""
         self.system.status.background_task = True
-        await self.hass.async_add_executor_job(setup_extra_stores, self)
+        await self.hass.async_add_executor_job(setup_extra_stores)
         self.hass.bus.async_fire("hacs/status", {})
         self.logger.debug(self.github.ratelimits.remaining)
         self.logger.debug(self.github.ratelimits.reset_utc)
@@ -195,7 +158,7 @@ class Hacs:
         await self.handle_critical_repositories_startup()
         await self.handle_critical_repositories()
         await self.load_known_repositories()
-        await self.clear_out_blacklisted_repositories()
+        await self.clear_out_removed_repositories()
 
         self.recuring_tasks.append(
             async_track_time_interval(
@@ -257,7 +220,8 @@ class Hacs:
         stored_critical = []
 
         for repository in critical:
-            self.common.blacklist.append(repository["repository"])
+            removed_repo = get_removed(repository["repository"])
+            removed_repo.removal_type = "critical"
             repo = self.get_by_name(repository["repository"])
 
             stored = {
@@ -266,7 +230,6 @@ class Hacs:
                 "link": repository["link"],
                 "acknowledged": True,
             }
-
             if repository["repository"] not in instored:
                 if repo is not None and repo.installed:
                     self.logger.critical(
@@ -278,6 +241,7 @@ class Hacs:
                     repo.remove()
                     await repo.uninstall()
             stored_critical.append(stored)
+            removed_repo.update_data(stored)
 
         # Save to FS
         await async_save_to_store(self.hass, "critical", stored_critical)
@@ -299,7 +263,7 @@ class Hacs:
         for repository in self.repositories:
             if (
                 repository.status.installed
-                and repository.category in self.common.categories
+                and repository.data.category in self.common.categories
             ):
                 self.factory.tasks.append(self.factory.safe_update(repository))
 
@@ -313,33 +277,35 @@ class Hacs:
     async def recuring_tasks_all(self, notarealarg=None):
         """Recuring tasks for all repositories."""
         self.logger.debug("Starting recuring background task for all repositories")
+        await self.hass.async_add_executor_job(setup_extra_stores)
         self.system.status.background_task = True
         self.hass.bus.async_fire("hacs/status", {})
         self.logger.debug(self.github.ratelimits.remaining)
         self.logger.debug(self.github.ratelimits.reset_utc)
         for repository in self.repositories:
-            if repository.category in self.common.categories:
+            if repository.data.category in self.common.categories:
                 self.factory.tasks.append(self.factory.safe_common_update(repository))
 
         await self.factory.execute()
         await self.load_known_repositories()
-        await self.clear_out_blacklisted_repositories()
+        await self.clear_out_removed_repositories()
         self.system.status.background_task = False
         await self.data.async_write()
         self.hass.bus.async_fire("hacs/status", {})
         self.hass.bus.async_fire("hacs/repository", {"action": "reload"})
         self.logger.debug("Recuring background task for all repositories done")
 
-    async def clear_out_blacklisted_repositories(self):
+    async def clear_out_removed_repositories(self):
         """Clear out blaclisted repositories."""
         need_to_save = False
-        for repository in self.common.blacklist:
-            if self.is_known(repository):
-                repository = self.get_by_name(repository)
-                if repository.status.installed:
+        for removed in removed_repositories:
+            if self.is_known(removed.repository):
+                repository = self.get_by_name(removed.repository)
+                if repository.status.installed and removed.removal_type != "critical":
                     self.logger.warning(
-                        f"You have {repository.information.full_name} installed with HACS "
-                        + "this repository has been blacklisted, please consider removing it."
+                        f"You have {repository.data.full_name} installed with HACS "
+                        + f"this repository has been removed, please consider removing it. "
+                        + f"Removal reason ({removed.removal_type})"
                     )
                 else:
                     need_to_save = True
@@ -353,7 +319,7 @@ class Hacs:
         repositories = {}
         for category in self.common.categories:
             repositories[category] = await get_default_repos_lists(
-                self.github, category
+                self.session, self.configuration.token, category
             )
             org = await get_default_repos_orgs(self.github, category)
             for repo in org:
@@ -370,17 +336,20 @@ class Hacs:
         self.logger.info("Loading known repositories")
         repositories = await self.get_repositories()
 
-        for item in await get_default_repos_lists(self.github, "blacklist"):
-            if item not in self.common.blacklist:
-                self.common.blacklist.append(item)
+        for item in await get_default_repos_lists(
+            self.session, self.configuration.token, "removed"
+        ):
+            removed = get_removed(item["repository"])
+            removed.reason = item.get("reason")
+            removed.link = item.get("link")
+            removed.removal_type = item.get("removal_type")
 
         for category in repositories:
             for repo in repositories[category]:
-                if repo in self.common.blacklist:
+                if is_removed(repo):
                     continue
                 if self.is_known(repo):
                     continue
-                self.factory.tasks.append(
-                    self.factory.safe_register(self, repo, category)
-                )
+                self.factory.tasks.append(self.factory.safe_register(repo, category))
         await self.factory.execute()
+        self.logger.info("Loading known repositories finished")

config/custom_components/hacs/hacsbase/backup.py

@@ -70,3 +70,48 @@ class Backup:
             while os.path.exists(self.backup_path):
                 sleep(0.1)
             self.logger.debug(f"Backup dir {self.backup_path} cleared")
+
+
+class BackupNetDaemon:
+    """BackupNetDaemon."""
+
+    def __init__(self, repository):
+        """Initialize."""
+        self.repository = repository
+        self.logger = Logger("hacs.backup")
+        self.backup_path = (
+            tempfile.gettempdir() + "/hacs_persistent_netdaemon/" + repository.data.name
+        )
+
+    def create(self):
+        """Create a backup in /tmp"""
+        if os.path.exists(self.backup_path):
+            shutil.rmtree(self.backup_path)
+            while os.path.exists(self.backup_path):
+                sleep(0.1)
+        os.makedirs(self.backup_path, exist_ok=True)
+
+        for filename in os.listdir(self.repository.content.path.local):
+            if filename.endswith(".yaml"):
+                source_file_name = f"{self.repository.content.path.local}/{filename}"
+                target_file_name = f"{self.backup_path}/{filename}"
+                shutil.copyfile(source_file_name, target_file_name)
+
+    def restore(self):
+        """Create a backup in /tmp"""
+        if os.path.exists(self.backup_path):
+            for filename in os.listdir(self.backup_path):
+                if filename.endswith(".yaml"):
+                    source_file_name = f"{self.backup_path}/{filename}"
+                    target_file_name = (
+                        f"{self.repository.content.path.local}/{filename}"
+                    )
+                    shutil.copyfile(source_file_name, target_file_name)
+
+    def cleanup(self):
+        """Create a backup in /tmp"""
+        if os.path.exists(self.backup_path):
+            shutil.rmtree(self.backup_path)
+            while os.path.exists(self.backup_path):
+                sleep(0.1)
+            self.logger.debug(f"Backup dir {self.backup_path} cleared")

config/custom_components/hacs/hacsbase/configuration.py

@@ -11,6 +11,8 @@ class Configuration:
     # Main configuration:
     appdaemon_path: str = "appdaemon/apps/"
     appdaemon: bool = False
+    netdaemon_path: str = "netdaemon/apps/"
+    netdaemon: bool = False
     config: dict = {}
     config_entry: dict = {}
     config_type: str = None

config/custom_components/hacs/hacsbase/data.py

@@ -1,50 +1,57 @@
 """Data handler for HACS."""
 from integrationhelper import Logger
-from . import Hacs
 from ..const import VERSION
 from ..repositories.repository import HacsRepository
 from ..repositories.manifest import HacsManifest
 from ..store import async_save_to_store, async_load_from_store
 
+from custom_components.hacs.globals import get_hacs, removed_repositories, get_removed
+from custom_components.hacs.helpers.register_repository import register_repository
 
-class HacsData(Hacs):
+
+class HacsData:
     """HacsData class."""
 
     def __init__(self):
         """Initialize."""
         self.logger = Logger("hacs.data")
+        self.hacs = get_hacs()
 
     async def async_write(self):
         """Write content to the store files."""
-        if self.system.status.background_task or self.system.disabled:
+        if self.hacs.system.status.background_task or self.hacs.system.disabled:
             return
 
         self.logger.debug("Saving data")
 
         # Hacs
         await async_save_to_store(
-            self.hass,
+            self.hacs.hass,
             "hacs",
             {
-                "view": self.configuration.frontend_mode,
-                "compact": self.configuration.frontend_compact,
-                "onboarding_done": self.configuration.onboarding_done,
+                "view": self.hacs.configuration.frontend_mode,
+                "compact": self.hacs.configuration.frontend_compact,
+                "onboarding_done": self.hacs.configuration.onboarding_done,
             },
         )
 
+        await async_save_to_store(
+            self.hacs.hass, "removed", [x.__dict__ for x in removed_repositories]
+        )
+
         # Repositories
         content = {}
-        for repository in self.repositories:
+        for repository in self.hacs.repositories:
             if repository.repository_manifest is not None:
                 repository_manifest = repository.repository_manifest.manifest
             else:
                 repository_manifest = None
             content[repository.information.uid] = {
-                "authors": repository.information.authors,
-                "category": repository.information.category,
-                "description": repository.information.description,
-                "downloads": repository.releases.last_release_object_downloads,
-                "full_name": repository.information.full_name,
+                "authors": repository.data.authors,
+                "category": repository.data.category,
+                "description": repository.data.description,
+                "downloads": repository.releases.downloads,
+                "full_name": repository.data.full_name,
                 "first_install": repository.status.first_install,
                 "hide": repository.status.hide,
                 "installed_commit": repository.versions.installed_commit,
@@ -52,54 +59,56 @@ class HacsData(Hacs):
                 "last_commit": repository.versions.available_commit,
                 "last_release_tag": repository.versions.available,
                 "last_updated": repository.information.last_updated,
-                "name": repository.information.name,
+                "name": repository.data.name,
                 "new": repository.status.new,
                 "repository_manifest": repository_manifest,
                 "selected_tag": repository.status.selected_tag,
                 "show_beta": repository.status.show_beta,
-                "stars": repository.information.stars,
-                "topics": repository.information.topics,
+                "stars": repository.data.stargazers_count,
+                "topics": repository.data.topics,
                 "version_installed": repository.versions.installed,
             }
 
-        await async_save_to_store(self.hass, "repositories", content)
-        self.hass.bus.async_fire("hacs/repository", {})
-        self.hass.bus.fire("hacs/config", {})
+        await async_save_to_store(self.hacs.hass, "repositories", content)
+        self.hacs.hass.bus.async_fire("hacs/repository", {})
+        self.hacs.hass.bus.fire("hacs/config", {})
 
     async def restore(self):
         """Restore saved data."""
-        hacs = await async_load_from_store(self.hass, "hacs")
-        repositories = await async_load_from_store(self.hass, "repositories")
+        hacs = await async_load_from_store(self.hacs.hass, "hacs")
+        repositories = await async_load_from_store(self.hacs.hass, "repositories")
+        removed = await async_load_from_store(self.hacs.hass, "removed")
         try:
             if not hacs and not repositories:
                 # Assume new install
-                self.system.status.new = True
+                self.hacs.system.status.new = True
                 return True
             self.logger.info("Restore started")
 
             # Hacs
-            self.configuration.frontend_mode = hacs.get("view", "Grid")
-            self.configuration.frontend_compact = hacs.get("compact", False)
-            self.configuration.onboarding_done = hacs.get("onboarding_done", False)
+            self.hacs.configuration.frontend_mode = hacs.get("view", "Grid")
+            self.hacs.configuration.frontend_compact = hacs.get("compact", False)
+            self.hacs.configuration.onboarding_done = hacs.get("onboarding_done", False)
+
+            for entry in removed:
+                removed_repo = get_removed(entry["repository"])
+                removed_repo.update_data(entry)
 
             # Repositories
             for entry in repositories:
                 repo = repositories[entry]
-                if repo["full_name"] == "hacs/integration":
-                    # Skip the old repo location
-                    continue
-                if not self.is_known(repo["full_name"]):
-                    await self.register_repository(
+                if not self.hacs.is_known(repo["full_name"]):
+                    await register_repository(
                         repo["full_name"], repo["category"], False
                     )
-                repository = self.get_by_name(repo["full_name"])
+                repository = self.hacs.get_by_name(repo["full_name"])
                 if repository is None:
                     self.logger.error(f"Did not find {repo['full_name']}")
                     continue
 
                 # Restore repository attributes
                 repository.information.uid = entry
-                await self.hass.async_add_executor_job(
+                await self.hacs.hass.async_add_executor_job(
                     restore_repository_data, repository, repo
                 )
 
@@ -114,13 +123,12 @@ def restore_repository_data(
     repository: type(HacsRepository), repository_data: dict
 ) -> None:
     """Restore Repository Data"""
-    repository.information.authors = repository_data.get("authors", [])
-    repository.information.description = repository_data.get("description")
-    repository.information.name = repository_data.get("name")
+    repository.data.authors = repository_data.get("authors", [])
+    repository.data.description = repository_data.get("description")
     repository.releases.last_release_object_downloads = repository_data.get("downloads")
     repository.information.last_updated = repository_data.get("last_updated")
-    repository.information.topics = repository_data.get("topics", [])
-    repository.information.stars = repository_data.get("stars", 0)
+    repository.data.topics = repository_data.get("topics", [])
+    repository.data.stargazers_count = repository_data.get("stars", 0)
     repository.releases.last_release = repository_data.get("last_release_tag")
     repository.status.hide = repository_data.get("hide", False)
     repository.status.installed = repository_data.get("installed", False)

config/custom_components/hacs/hacsbase/exceptions.py

@@ -3,3 +3,7 @@
 
 class HacsException(Exception):
     """Super basic."""
+
+
+class HacsExpectedException(HacsException):
+    """For stuff that are expected."""

config/custom_components/hacs/hacsbase/task_factory.py

@@ -5,6 +5,10 @@ from datetime import timedelta
 import asyncio
 from aiogithubapi import AIOGitHubException
 
+from custom_components.hacs.hacsbase.exceptions import HacsException
+from custom_components.hacs.helpers.register_repository import register_repository
+
+
 max_concurrent_tasks = asyncio.Semaphore(15)
 sleeper = 5
 
@@ -44,8 +48,8 @@ class HacsTaskFactory:
         async with max_concurrent_tasks:
             try:
                 await repository.common_update()
-            except AIOGitHubException as exception:
-                logger.error(exception)
+            except (AIOGitHubException, HacsException) as exception:
+                logger.error("%s - %s", repository.data.full_name, exception)
 
             # Due to GitHub ratelimits we need to sleep a bit
             await asyncio.sleep(sleeper)
@@ -54,18 +58,18 @@ class HacsTaskFactory:
         async with max_concurrent_tasks:
             try:
                 await repository.update_repository()
-            except AIOGitHubException as exception:
-                logger.error(exception)
+            except (AIOGitHubException, HacsException) as exception:
+                logger.error("%s - %s", repository.data.full_name, exception)
 
             # Due to GitHub ratelimits we need to sleep a bit
             await asyncio.sleep(sleeper)
 
-    async def safe_register(self, hacs, repo, category):
+    async def safe_register(self, repo, category):
         async with max_concurrent_tasks:
             try:
-                await hacs.register_repository(repo, category)
-            except AIOGitHubException as exception:
-                logger.error(exception)
+                await register_repository(repo, category)
+            except (AIOGitHubException, HacsException) as exception:
+                logger.error("%s - %s", repo, exception)
 
             # Due to GitHub ratelimits we need to sleep a bit
             await asyncio.sleep(sleeper)