Firestore: 'test_query_watch_order' failure on CI account

[tseaver]

This one appears not to be flaky, but rather is failing because of the move to a new CI account for Firestore.

____________________________ test_watch_query_order ____________________________
client = <google.cloud.firestore_v1beta1.client.Client object at 0x7f4da96726d0>
cleanup = <built-in method append of list object at 0x7f4da20dfbd8>
    def test_watch_query_order(client, cleanup):
        db = client
        unique_id = unique_resource_id()
        doc_ref1 = db.collection(u"users").document(u"alovelace" + unique_id)
        doc_ref2 = db.collection(u"users").document(u"asecondlovelace" + unique_id)
        doc_ref3 = db.collection(u"users").document(u"athirdlovelace" + unique_id)
        doc_ref4 = db.collection(u"users").document(u"afourthlovelace" + unique_id)
        doc_ref5 = db.collection(u"users").document(u"afifthlovelace" + unique_id)
        query_ref = (
            db.collection(u"users")
            .where("first", "==", u"Ada" + unique_id)
            .order_by("last")
        )
        # Setup listener
        def on_snapshot(docs, changes, read_time):
            try:
                if len(docs) != 5:
                    return
                # A snapshot should return the same thing as if a query ran now.
                query_ran = query_ref.get()
                query_ran_results = [i for i in query_ran]
                assert len(docs) == len(query_ran_results)
                # compare the order things are returned
                for snapshot, query in zip(docs, query_ran_results):
                    assert snapshot.get("last") == query.get(
                        "last"
                    ), "expect the sort order to match, last"
                    assert snapshot.get("born") == query.get(
                        "born"
                    ), "expect the sort order to match, born"
                on_snapshot.called_count += 1
                on_snapshot.last_doc_count = len(docs)
            except Exception as e:
                on_snapshot.failed = e
        on_snapshot.called_count = 0
        on_snapshot.last_doc_count = 0
        on_snapshot.failed = None
        query_ref.on_snapshot(on_snapshot)
        sleep(1)
        doc_ref1.set({u"first": u"Ada" + unique_id, u"last": u"Lovelace", u"born": 1815})
        doc_ref2.set(
            {u"first": u"Ada" + unique_id, u"last": u"SecondLovelace", u"born": 1815}
        )
        doc_ref3.set(
            {u"first": u"Ada" + unique_id, u"last": u"ThirdLovelace", u"born": 1815}
        )
        doc_ref4.set(
            {u"first": u"Ada" + unique_id, u"last": u"FourthLovelace", u"born": 1815}
        )
        doc_ref5.set({u"first": u"Ada" + unique_id, u"last": u"lovelace", u"born": 1815})
        for _ in range(10):
            if on_snapshot.last_doc_count == 5:
                break
            sleep(1)
        if on_snapshot.failed:
            raise on_snapshot.failed
        if on_snapshot.last_doc_count != 5:
            raise AssertionError(
>               "5 docs expected in snapshot method " + str(on_snapshot.last_doc_count)
            )
E           AssertionError: 5 docs expected in snapshot method 0
tests/system.py:866: AssertionError
------------------------------ Captured log call -------------------------------
bidi.py                    547 ERROR    Thread-ConsumeBidirectionalStream caught unexpected exception Error 9:  The query requires an index. You can create it here: https://console.firebase.google.com/project/abstract-disk-227418/database/firestore/indexes?create_index=EgV1c2VycxoJCgVmaXJzdBACGggKBGxhc3QQAhoMCghfX25hbWVfXxAC and will exit.
Traceback (most recent call last):
  File "/tmpfs/src/github/google-cloud-python/api_core/google/api_core/bidi.py", line 531, in _thread_main
    self._on_response(response)
  File "/tmpfs/src/github/google-cloud-python/firestore/google/cloud/firestore_v1beta1/watch.py", line 441, in on_snapshot
    meth(proto)
  File "/tmpfs/src/github/google-cloud-python/firestore/google/cloud/firestore_v1beta1/watch.py", line 396, in _on_snapshot_target_change_remove
    raise RuntimeError(message)
RuntimeError: Error 9:  The query requires an index. You can create it here: https://console.firebase.google.com/project/abstract-disk-227418/database/firestore/indexes?create_index=EgV1c2VycxoJCgVmaXJzdBACGggKBGxhc3QQAhoMCghfX25hbWVfXxAC

[tseaver]

I've seen successful Firestore system tests, e.g.: this one for PR #7072.