diff --git a/tools/perf/util/symbol.c b/tools/perf/util/symbol.c
index a4e7459..b6ab23d 100644
--- a/tools/perf/util/symbol.c
+++ b/tools/perf/util/symbol.c
@@ -1572,7 +1572,7 @@
 		return -1;
 
 	dso__set_loaded(self, map->type);
-	err = dso__load_sym(self, map, session, self->long_name, fd, filter, 1, 0);
+	err = dso__load_sym(self, map, session, vmlinux, fd, filter, 1, 0);
 	close(fd);
 
 	return err;
@@ -1584,6 +1584,26 @@
 	int err;
 	const char *kallsyms_filename = NULL;
 	char *kallsyms_allocated_filename = NULL;
+	/*
+	 * Step 1: if the user specified a vmlinux filename, use it and only
+	 * it, reporting errors to the user if it cannot be used.
+	 *
+	 * For instance, try to analyse an ARM perf.data file _without_ a
+	 * build-id, or if the user specifies the wrong path to the right
+	 * vmlinux file, obviously we can't fallback to another vmlinux (a
+	 * x86_86 one, on the machine where analysis is being performed, say),
+	 * or worse, /proc/kallsyms.
+	 *
+	 * If the specified file _has_ a build-id and there is a build-id
+	 * section in the perf.data file, we will still do the expected
+	 * validation in dso__load_vmlinux and will bail out if they don't
+	 * match.
+	 */
+	if (symbol_conf.vmlinux_name != NULL) {
+		err = dso__load_vmlinux(self, map, session,
+					symbol_conf.vmlinux_name, filter);
+		goto out_try_fixup;
+	}
 
 	if (vmlinux_path != NULL) {
 		int i;
@@ -1618,46 +1638,41 @@
 				goto do_kallsyms;
 			}
 		}
-
+		/*
+		 * Now look if we have it on the build-id cache in
+		 * $HOME/.debug/[kernel.kallsyms].
+		 */
 		build_id__sprintf(self->build_id, sizeof(self->build_id),
 				  sbuild_id);
 
 		if (asprintf(&kallsyms_allocated_filename,
 			     "%s/.debug/[kernel.kallsyms]/%s",
-			     getenv("HOME"), sbuild_id) != -1) {
-			if (access(kallsyms_filename, F_OK)) {
-				kallsyms_filename = kallsyms_allocated_filename;
-				goto do_kallsyms;
-			}
-			free(kallsyms_allocated_filename);
-			kallsyms_allocated_filename = NULL;
-		}
-
-		goto do_vmlinux;
-	}
-
-	if (self->long_name[0] == '[') {
-		kallsyms_filename = "/proc/kallsyms";
-		goto do_kallsyms;
-	}
-
-do_vmlinux:
-	err = dso__load_vmlinux(self, map, session, self->long_name, filter);
-	if (err <= 0) {
-		if (self->has_build_id)
+			     getenv("HOME"), sbuild_id) == -1)
 			return -1;
 
-		pr_info("The file %s cannot be used, "
-			"trying to use /proc/kallsyms...", self->long_name);
-do_kallsyms:
-		err = dso__load_kallsyms(self, kallsyms_filename, map, session, filter);
-		if (err > 0 && kallsyms_filename == NULL)
-                        dso__set_long_name(self, strdup("[kernel.kallsyms]"));
-		free(kallsyms_allocated_filename);
+		if (access(kallsyms_filename, F_OK)) {
+			free(kallsyms_allocated_filename);
+			return -1;
+		}
+
+		kallsyms_filename = kallsyms_allocated_filename;
+	} else {
+		/*
+		 * Last resort, if we don't have a build-id and couldn't find
+		 * any vmlinux file, try the running kernel kallsyms table.
+		 */
+		kallsyms_filename = "/proc/kallsyms";
 	}
 
+do_kallsyms:
+	err = dso__load_kallsyms(self, kallsyms_filename, map, session, filter);
+	free(kallsyms_allocated_filename);
+
+out_try_fixup:
 	if (err > 0) {
 out_fixup:
+		if (kallsyms_filename == NULL)
+			dso__set_long_name(self, strdup("[kernel.kallsyms]"));
 		map__fixup_start(map);
 		map__fixup_end(map);
 	}