[bitbake-devel] [PATCH 1/4] hashserv: Add support for equivalent hash reporting

[akuster808]

On 12/4/19 3:52 AM, Richard Purdie wrote:
> The reason for this should be recorded in the commit logs. Imagine
> you have a target recipe (e.g. meta-extsdk-toolchain) which depends on
> gdb-cross. sstate in OE-Core allows gdb-cross to have the same hash
> regardless of whether its built on x86 or arm. The outhash will be
> different.
>
> We need hashequiv to be able to adapt to the prescence of sstate artefacts
> for meta-extsdk-toolchain and allow the hashes to re-intersect, rather than
> trying to force a rebuild of meta-extsdk-toolchain. By this point in the build,
> it would have already been installed from sstate so the build needs to adapt.
>
> Equivalent hashes should be reported to the server as a taskhash that
> needs to map to an specific unihash. This patch adds API to the hashserv
> client/server to allow this.
>
> [Thanks to Joshua Watt for help with this patch]

This sounds like 1.44 backport worthy?

- armin
>
> Signed-off-by: Richard Purdie <richard.purdie at linuxfoundation.org>
> ---
>  lib/hashserv/client.py |  8 ++++++++
>  lib/hashserv/server.py | 36 ++++++++++++++++++++++++++++++++++++
>  2 files changed, 44 insertions(+)
>
> diff --git a/lib/hashserv/client.py b/lib/hashserv/client.py
> index f65956617b..ae0cce9df4 100644
> --- a/lib/hashserv/client.py
> +++ b/lib/hashserv/client.py
> @@ -148,6 +148,14 @@ class Client(object):
>          m['unihash'] = unihash
>          return self.send_message({'report': m})
>  
> +    def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
> +        self._set_mode(self.MODE_NORMAL)
> +        m = extra.copy()
> +        m['taskhash'] = taskhash
> +        m['method'] = method
> +        m['unihash'] = unihash
> +        return self.send_message({'report-equiv': m})
> +
>      def get_stats(self):
>          self._set_mode(self.MODE_NORMAL)
>          return self.send_message({'get-stats': None})
> diff --git a/lib/hashserv/server.py b/lib/hashserv/server.py
> index 0aff77688e..cc7e48233b 100644
> --- a/lib/hashserv/server.py
> +++ b/lib/hashserv/server.py
> @@ -143,6 +143,7 @@ class ServerClient(object):
>              handlers = {
>                  'get': self.handle_get,
>                  'report': self.handle_report,
> +                'report-equiv': self.handle_equivreport,
>                  'get-stream': self.handle_get_stream,
>                  'get-stats': self.handle_get_stats,
>                  'reset-stats': self.handle_reset_stats,
> @@ -303,6 +304,41 @@ class ServerClient(object):
>  
>          self.write_message(d)
>  
> +    async def handle_equivreport(self, data):
> +        with closing(self.db.cursor()) as cursor:
> +            insert_data = {
> +                'method': data['method'],
> +                'outhash': "",
> +                'taskhash': data['taskhash'],
> +                'unihash': data['unihash'],
> +                'created': datetime.now()
> +            }
> +
> +            for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
> +                if k in data:
> +                    insert_data[k] = data[k]
> +
> +            cursor.execute('''INSERT OR IGNORE INTO tasks_v2 (%s) VALUES (%s)''' % (
> +                ', '.join(sorted(insert_data.keys())),
> +                ', '.join(':' + k for k in sorted(insert_data.keys()))),
> +                insert_data)
> +
> +            self.db.commit()
> +
> +            # Fetch the unihash that will be reported for the taskhash. If the
> +            # unihash matches, it means this row was inserted (or the mapping
> +            # was already valid)
> +            row = self.query_equivalent(data['method'], data['taskhash'])
> +
> +            if row['unihash'] == data['unihash']:
> +                logger.info('Adding taskhash equivalence for %s with unihash %s',
> +                                data['taskhash'], row['unihash'])
> +
> +            d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
> +
> +        self.write_message(d)
> +
> +
>      async def handle_get_stats(self, request):
>          d = {
>              'requests': self.request_stats.todict(),