{ "schema_version": "1.0", "record_type": "research_findings", "language": "en", "source": { "title": "Does AI Recommend the Same Hotels Five Weeks Later?", "publisher": "Americas Great Resorts", "author": "Andrew Paul", "published": "2026-11-02", "url": "https://www.americasgreatresorts.net/ai-hotel-recommendations-five-weeks-later/", "publisher_disclosure": "Americas Great Resorts provides hotel marketing and AI visibility services." }, "study_scope": { "collection_dates": ["2026-09-27", "2026-11-02"], "traveler_questions": 100, "destinations": 10, "questions_per_destination": 10, "shared_question_types": [ "general discovery", "luxury", "couples", "families", "boutique hotels", "spas", "dining", "service" ], "local_questions_per_destination": 2, "engines": ["ChatGPT", "Google AI Mode", "Gemini", "Copilot"], "september_repetitions": 3, "november_repetitions": 1, "september_substantive_answers": 1177, "november_substantive_answers": 400, "total_question_engine_groups": 400, "eligible_question_engine_groups": 379, "excluded_question_engine_groups": 21, "september_collection_errors": 9, "september_nonanswers": 14, "september_unusable_responses": 23, "exclusion_reason": "The 21 excluded groups lacked three substantive September answers. Their available substantive answers remain included in destination appearance counts.", "hotel_set_match_definition": "The same identified hotels appear in both answers. Order, wording, and repeated mentions are ignored. Adding or removing one hotel changes the set.", "qualifying_appearance_definition": "An identifiable hotel featured in an answer counts at most once per answer. Clear name variants are consolidated. Separately named hotels remain distinct, and explicit joint recommendations credit each identifiable hotel. A clearly identified hotel spa or relevant on-site amenity contributes to the parent hotel's appearance. Citation-only names, ancillary cards, incidental mentions, and ambiguous identities are excluded.", "destination_leader_definition": "The hotel with the most qualifying appearances across the destination's questions and engines, calculated separately for each collection date.", "appearance_rate_definition": "Qualifying appearances for a property divided by the assessed answers for that destination and collection date.", "extraction": "The follow-up reused September's publication conventions and name decisions. It used AI-assisted extraction followed by review of flagged identities and supporting excerpts. The extraction workflows differed between editions." }, "findings": [ { "id": "hotel-recommendations-five-weeks#matched-reference-nonmatches", "kind": "finding", "topic": "hotel-recommendation-variation", "claim": "Hotel sets frequently differed on the same day as well as five weeks later.", "evidence": { "september_reference_repetitions": [1, 2], "same_day_september": { "answer_tested": "September repetition 3", "matched_neither_reference": 317, "total": 379, "percent": 83.6 }, "five_weeks_later": { "answer_tested": "November answer", "matched_neither_reference": 329, "total": 379, "percent": 86.8 }, "gap_range_across_reference_pairs_percentage_points": [3.2, 6.9], "identical_across_all_three_september_repetitions": { "groups": 16, "total": 379, "groups_with_at_least_one_membership_change": 363 } }, "exhibit_ids": ["exhibit-2", "exhibit-3", "exhibit-4"], "source_location": "Repeating the same question on the same day produced different hotel lists", "limitation": "The comparison does not isolate elapsed time as the cause. A set can differ by a single property." }, { "id": "hotel-recommendations-five-weeks#three-reference-matches", "kind": "finding", "topic": "hotel-recommendation-repeatability", "claim": "61 of 379 November hotel sets matched at least one of the three September sets for the same question and engine.", "evidence": { "september_reference_repetitions": [1, 2, 3], "matched_at_least_one_reference": 61, "matched_no_reference": 318, "total": 379, "percent": 16.1, "nonmatch_percent": 83.9, "copilot": {"matches": 47, "total": 100, "percent": 47.0}, "chatgpt_google_ai_mode_and_gemini": { "matches": 14, "total": 279, "percent": 5.0 } }, "source_location": "Copilot reproduced earlier lineups more often", "limitation": "This comparison uses three September references. The matched same-day comparison uses two. Copilot already had a lower same-day nonmatch rate in the two-reference comparison." }, { "id": "hotel-recommendations-five-weeks#destination-leadership", "kind": "finding", "topic": "destination-leadership", "claim": "Seven of ten destinations kept the same leading hotel across the two collections.", "evidence": { "destinations_with_same_leader": 7, "destinations_assessed": 10, "destinations_with_new_leader": ["Miami area", "Punta Cana", "Los Cabos"], "november_leaders_with_one_appearance_margin": 1, "november_leaders_with_two_appearance_margin": 5 }, "exhibit_ids": ["exhibit-1"], "source_location": "7 destinations kept the same leading hotel", "limitation": "Destination counts combine engines. The leader need not lead every engine, and small margins may change in another collection." }, { "id": "hotel-recommendations-five-weeks#identified-hotels-per-answer", "kind": "finding", "topic": "identified-hotel-list-length", "claim": "November answers contained more identified hotels on average than September answers.", "evidence": { "september": { "qualifying_appearances": 6110, "assessed_answers": 1177, "mean_identified_hotels_per_answer": 5.19 }, "november": { "qualifying_appearances": 2342, "assessed_answers": 400, "mean_identified_hotels_per_answer": 5.86 }, "approximate_percent_increase": 13, "engine_level_observation": "ChatGPT and Gemini had longer identified lists and higher November nonmatch rates in the main matched-reference comparison. Google AI Mode changed little on both measures. Copilot's mean list length and nonmatch rate were lower in November." }, "exhibit_ids": ["exhibit-2", "exhibit-5", "exhibit-6"], "source_location": "November's identified lists were longer overall", "limitation": "Extraction workflows differed. The observed association does not establish that list length caused the nonmatch differences or separate changes in generated answers from changes in measurement." }, { "id": "hotel-recommendations-five-weeks#appearance-rate-versus-rank", "kind": "finding", "topic": "property-appearance-rate-and-rank", "claim": "Tortuga Bay's appearance rate rose while its Punta Cana rank fell from first to third.", "evidence": { "property": "Tortuga Bay", "destination": "Punta Cana", "september": {"appearances": 56, "answers": 117, "percent": 47.9, "rank": 1}, "november": {"appearances": 21, "answers": 40, "percent": 52.5, "rank": 3}, "november_properties_ahead": [ {"property": "Secrets Cap Cana Resort & Spa", "appearances": 26, "answers": 40, "percent": 65.0}, {"property": "Eden Roc Cap Cana", "appearances": 24, "answers": 40, "percent": 60.0} ] }, "exhibit_ids": ["exhibit-1"], "source_location": "Rank and appearance rate moved differently", "limitation": "The comparison explains the change in relative position within the observed answers. It does not establish why any engine selected a property or whether the change persisted." } ], "exhibits": { "exhibit-1": { "title": "Destination leaders and November runners-up", "november_answers_per_destination": 40, "september_answers_per_destination_range": [116, 120], "rows": [ { "destination": "New York City", "september_leader": "Mandarin Oriental, New York", "november_leader": {"property": "Mandarin Oriental, New York", "appearances": 18}, "november_runners_up": [{"property": "The Peninsula New York", "appearances": 16}] }, { "destination": "Miami area", "september_leader": "The Setai, Miami Beach", "november_leader": {"property": "Faena Miami Beach", "appearances": 25}, "november_runners_up": [{"property": "Four Seasons Hotel at The Surf Club", "appearances": 20}] }, { "destination": "Las Vegas", "september_leader": "Wynn Las Vegas", "november_leader": {"property": "Wynn Las Vegas", "appearances": 26}, "november_runners_up": [ {"property": "Four Seasons Hotel Las Vegas", "appearances": 24}, {"property": "The Venetian", "appearances": 24} ] }, { "destination": "Los Angeles area", "september_leader": "Hotel Bel-Air", "november_leader": {"property": "Hotel Bel-Air", "appearances": 22}, "november_runners_up": [{"property": "The Beverly Hills Hotel", "appearances": 21}] }, { "destination": "Orlando area", "september_leader": "Four Seasons Orlando", "november_leader": {"property": "Four Seasons Orlando", "appearances": 32}, "november_runners_up": [{"property": "Ritz-Carlton Orlando, Grande Lakes", "appearances": 30}] }, { "destination": "Chicago", "september_leader": "The Peninsula Chicago", "november_leader": {"property": "The Peninsula Chicago", "appearances": 32}, "november_runners_up": [{"property": "Four Seasons Hotel Chicago", "appearances": 29}] }, { "destination": "Cancún / Costa Mujeres", "september_leader": "Excellence Playa Mujeres", "november_leader": {"property": "Excellence Playa Mujeres", "appearances": 23}, "november_runners_up": [{"property": "Atelier Playa Mujeres", "appearances": 21}] }, { "destination": "Punta Cana", "september_leader": "Tortuga Bay", "november_leader": {"property": "Secrets Cap Cana Resort & Spa", "appearances": 26}, "november_runners_up": [{"property": "Eden Roc Cap Cana", "appearances": 24}] }, { "destination": "Los Cabos", "september_leader": "Waldorf Astoria Los Cabos Pedregal", "november_leader": {"property": "One&Only Palmilla", "appearances": 28}, "november_runners_up": [{"property": "Waldorf Astoria Los Cabos Pedregal", "appearances": 24}] }, { "destination": "Honolulu / Waikīkī", "september_leader": "Halekulani", "november_leader": {"property": "Halekulani", "appearances": 33}, "november_runners_up": [{"property": "The Kahala", "appearances": 28}] } ] }, "exhibit-2": { "title": "Same-day and five-week nonmatches against the same two September references", "september_reference_repetitions": [1, 2], "september_repetition_tested": 3, "rows": [ {"engine": "ChatGPT", "eligible_groups": 91, "september_nonmatches": 77, "september_nonmatch_percent": 84.6, "november_nonmatches": 90, "november_nonmatch_percent": 98.9}, {"engine": "Google AI Mode", "eligible_groups": 91, "september_nonmatches": 85, "september_nonmatch_percent": 93.4, "november_nonmatches": 84, "november_nonmatch_percent": 92.3}, {"engine": "Gemini", "eligible_groups": 97, "september_nonmatches": 86, "september_nonmatch_percent": 88.7, "november_nonmatches": 96, "november_nonmatch_percent": 99.0}, {"engine": "Copilot", "eligible_groups": 100, "september_nonmatches": 69, "september_nonmatch_percent": 69.0, "november_nonmatches": 59, "november_nonmatch_percent": 59.0}, {"engine": "All engines", "eligible_groups": 379, "september_nonmatches": 317, "september_nonmatch_percent": 83.6, "november_nonmatches": 329, "november_nonmatch_percent": 86.8} ] }, "exhibit-3": { "title": "Hotel-set nonmatches varied by engine on the same-day and five-week checks", "data_exhibit_id": "exhibit-2" }, "exhibit-4": { "title": "Each held-out September answer and the November answer tested against the same reference pair", "gap_unit": "percentage points", "gap_calculation": "November nonmatch rate minus the paired September nonmatch rate, computed from the unrounded fractions.", "rows": [ {"september_repetition_held_out": 1, "reference_repetitions": [2, 3], "eligible_groups": 379, "september_nonmatches": 303, "september_nonmatch_percent": 79.9, "november_nonmatches": 329, "november_nonmatch_percent": 86.8, "gap_percentage_points": 6.9}, {"september_repetition_held_out": 2, "reference_repetitions": [1, 3], "eligible_groups": 379, "september_nonmatches": 307, "september_nonmatch_percent": 81.0, "november_nonmatches": 332, "november_nonmatch_percent": 87.6, "gap_percentage_points": 6.6}, {"september_repetition_held_out": 3, "reference_repetitions": [1, 2], "eligible_groups": 379, "september_nonmatches": 317, "september_nonmatch_percent": 83.6, "november_nonmatches": 329, "november_nonmatch_percent": 86.8, "gap_percentage_points": 3.2} ] }, "exhibit-5": { "title": "Mean identified hotels per assessed answer by engine and collection date", "unit": "identified hotels per assessed answer", "september_assessed_answers_all_engines": 1177, "november_assessed_answers_all_engines": 400, "rows": [ {"engine": "ChatGPT", "september_mean": 6.82, "november_mean": 7.52}, {"engine": "Google AI Mode", "september_mean": 4.83, "november_mean": 4.89}, {"engine": "Gemini", "september_mean": 4.87, "november_mean": 6.80}, {"engine": "Copilot", "september_mean": 4.28, "november_mean": 4.21}, {"engine": "All engines", "september_mean": 5.19, "november_mean": 5.86} ] }, "exhibit-6": { "title": "Identified list length increased for ChatGPT and Gemini", "data_exhibit_id": "exhibit-5" } }, "property_examples": [ { "property": "The Peninsula Chicago", "destination": "Chicago", "september": {"appearances": 96, "answers": 118, "percent": 81.4, "rank": 1}, "november": {"appearances": 32, "answers": 40, "percent": 80.0, "rank": 1} }, { "property": "Halekulani", "destination": "Honolulu / Waikīkī", "september": {"appearances": 91, "answers": 118, "percent": 77.1, "rank": 1}, "november": {"appearances": 33, "answers": 40, "percent": 82.5, "rank": 1} }, { "property": "The Setai, Miami Beach", "destination": "Miami area", "september": {"appearances": 51, "answers": 117, "percent": 43.6, "rank": 1}, "november": {"appearances": 17, "answers": 40, "percent": 42.5} }, { "property": "Faena Miami Beach", "destination": "Miami area", "november": {"appearances": 25, "answers": 40, "percent": 62.5, "rank": 1} }, { "property": "Tortuga Bay", "destination": "Punta Cana", "september": {"appearances": 56, "answers": 117, "percent": 47.9, "rank": 1}, "november": {"appearances": 21, "answers": 40, "percent": 52.5, "rank": 3} }, { "property": "Secrets Cap Cana Resort & Spa", "destination": "Punta Cana", "september": {"appearances": 49, "answers": 117, "percent": 41.9}, "november": {"appearances": 26, "answers": 40, "percent": 65.0, "rank": 1} }, { "property": "Eden Roc Cap Cana", "destination": "Punta Cana", "september": {"appearances": 53, "answers": 117, "percent": 45.3}, "november": {"appearances": 24, "answers": 40, "percent": 60.0, "rank": 2} } ], "hotel_set_comparison_example": { "engine": "ChatGPT", "destination": "Chicago", "september_identified_hotels": 6, "november_retained_all_six": true, "november_additional_hotel": "Pendry Chicago", "exact_hotel_set_match": false, "source_location": "Repeating the same question on the same day produced different hotel lists" }, "interpretation_limits": [ "The study is observational and describes the tested questions and saved answers under the published counting rules.", "November's single pass cannot establish lasting gains or losses.", "Exact model versions, session conditions, and internal retrieval processes were not uniformly controlled.", "The study does not isolate elapsed time as the cause of differences, rank the quality of engine advice, verify hotel operations or amenities, or measure bookings.", "The study measures hotel-set reproducibility and property appearance rates separately." ], "related_research": [ { "title": "September 10-destination study", "relationship": "Baseline for this follow-up", "collection_date": "2026-09-27", "substantive_answers": 1177, "qualifying_hotel_appearances": 6110 }, { "title": "AGR Luxury Hotel AI Visibility Index", "relationship": "Separate collection with a different question set and counting method", "collection_date": "2026-07-29", "markets": 6, "recommendation_slots": 824, "pool_with_this_study": false } ] }