qiurui/armbian-build
1
0
Fork 0
mirror of https://github.com/armbian/build.git synced 2026-09-08 23:59:54 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
153d2ff622dc8787c2c34be3af8d880595b382c7
armbian-build/patch/kernel/archive/uefi-x86-7.2/1001-Add-t2bce-driver-stack.patch
T
Ricardo Pardini 153d2ff622 uefi-x86-7.2: update applet2 patches from t2linux/linux-t2-patches@1637df4 
- rewritten/attribution preserved
- rewrite .config with new t2 stuff +recover SOUND/SND/SND_SOC
2026-09-08 17:22:28 +02:00

8452 lines
299 KiB
Diff
Raw Blame History

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
<41898282+github-actions[bot]@users.noreply.github.com>
Date: Fri, 7 Aug 2026 15:55:34 +0000
Subject: Add t2bce driver stack
---
drivers/staging/t2bce/Kconfig | 40 +
drivers/staging/t2bce/Makefile | 5 +
drivers/staging/t2bce/t2bce_audio/Makefile | 7 +
drivers/staging/t2bce/t2bce_audio/audio.c | 930 +++++++
drivers/staging/t2bce/t2bce_audio/audio.h | 148 ++
drivers/staging/t2bce/t2bce_audio/description.h | 42 +
drivers/staging/t2bce/t2bce_audio/pcm.c | 520 ++++
drivers/staging/t2bce/t2bce_audio/pcm.h | 17 +
drivers/staging/t2bce/t2bce_audio/protocol.c | 347 +++
drivers/staging/t2bce/t2bce_audio/protocol.h | 147 ++
drivers/staging/t2bce/t2bce_audio/protocol_bce.c | 260 ++
drivers/staging/t2bce/t2bce_audio/protocol_bce.h | 72 +
drivers/staging/t2bce/t2bce_core/Makefile | 7 +
drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h | 83 +
drivers/staging/t2bce/t2bce_core/mailbox.c | 234 ++
drivers/staging/t2bce/t2bce_core/mailbox.h | 64 +
drivers/staging/t2bce/t2bce_core/t2bce.h | 51 +
drivers/staging/t2bce/t2bce_core/t2bce_main.c | 766 ++++++
drivers/staging/t2bce/t2bce_core/transport.c | 408 +++
drivers/staging/t2bce/t2bce_dma/Makefile | 6 +
drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h | 182 ++
drivers/staging/t2bce/t2bce_dma/queue.c | 625 +++++
drivers/staging/t2bce/t2bce_vhci/Makefile | 7 +
drivers/staging/t2bce/t2bce_vhci/command.h | 206 ++
drivers/staging/t2bce/t2bce_vhci/queue.c | 269 ++
drivers/staging/t2bce/t2bce_vhci/queue.h | 76 +
drivers/staging/t2bce/t2bce_vhci/transfer.c | 1261 +++++++++
drivers/staging/t2bce/t2bce_vhci/transfer.h | 86 +
drivers/staging/t2bce/t2bce_vhci/vhci.c | 1293 ++++++++++
drivers/staging/t2bce/t2bce_vhci/vhci.h | 69 +
30 files changed, 8228 insertions(+)
diff --git a/drivers/staging/t2bce/Kconfig b/drivers/staging/t2bce/Kconfig
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/Kconfig
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: GPL-2.0-only
+menu "Apple T2 Buffer Copy Engine support"
+
+config T2BCE_DMA
+ tristate
+ help
+ DMA queue support for the Apple T2 Buffer Copy Engine.
+
+config T2BCE_CORE
+ tristate "Apple T2 Buffer Copy Engine core"
+ depends on X86_64 && PCI
+ select T2BCE_DMA
+ help
+ Core transport driver for the Buffer Copy Engine in Apple T2 Macs.
+ It provides the mailbox, DMA queue coordination, power management,
+ and transport API used by the T2 VHCI and audio drivers.
+
+ If compiled as a module, it will be called t2bce_core.
+
+config T2BCE_VHCI
+ tristate "Apple T2 virtual USB host controller"
+ depends on T2BCE_CORE && USB
+ help
+ Virtual USB 2.0 host controller for devices exposed by bridgeOS,
+ including the internal keyboard, trackpad, Touch Bar, and camera.
+
+ If compiled as a module, it will be called t2bce_vhci.
+
+config T2BCE_AUDIO
+ tristate "Apple T2 audio support"
+ depends on T2BCE_CORE && SND
+ select SND_PCM
+ select SND_JACK
+ help
+ ALSA driver for the audio endpoints exposed by bridgeOS on Apple
+ T2 Macs.
+
+ If compiled as a module, it will be called t2bce_audio.
+
+endmenu
diff --git a/drivers/staging/t2bce/Makefile b/drivers/staging/t2bce/Makefile
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_T2BCE_DMA) += t2bce_dma/
+obj-$(CONFIG_T2BCE_CORE) += t2bce_core/
+obj-$(CONFIG_T2BCE_VHCI) += t2bce_vhci/
+obj-$(CONFIG_T2BCE_AUDIO) += t2bce_audio/
diff --git a/drivers/staging/t2bce/t2bce_audio/Makefile b/drivers/staging/t2bce/t2bce_audio/Makefile
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_T2BCE_AUDIO) += t2bce_audio.o
+t2bce_audio-y := audio.o protocol.o protocol_bce.o pcm.o
+
+ccflags-y += -DWITHOUT_NVME_PATCH
+ccflags-y += -I$(src)
+ccflags-y += -I$(src)/../t2bce_core/include
diff --git a/drivers/staging/t2bce/t2bce_audio/audio.c b/drivers/staging/t2bce/t2bce_audio/audio.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/audio.c
@@ -0,0 +1,930 @@
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/err.h>
+#include <sound/core.h>
+#include <sound/initval.h>
+#include <sound/pcm.h>
+#include <sound/jack.h>
+#include "audio.h"
+#include "pcm.h"
+#include <linux/version.h>
+
+static int t2audio_alsa_index = SNDRV_DEFAULT_IDX1;
+static char *t2audio_alsa_id = SNDRV_DEFAULT_STR1;
+
+static dev_t t2audio_chrdev;
+static struct class *t2audio_class;
+
+static int t2audio_init_cmd(struct t2audio_device *a);
+static int t2audio_init_bs(struct t2audio_device *a);
+static void t2audio_init_dev(struct t2audio_device *a, t2audio_device_id_t dev_id);
+static void t2audio_free_dev(struct t2audio_subdevice *sdev);
+static void t2audio_reset_stream(struct t2audio_stream *stream);
+static void t2audio_reset_streams(struct t2audio_device *a);
+static void t2audio_reset_clock(struct t2audio_device *a);
+static void t2audio_resume_work(struct work_struct *ws);
+static void t2audio_resume_complete(void *userdata);
+static void t2audio_pm_prepare_client(void *userdata);
+static void t2audio_pm_shutdown_client(void *userdata);
+
+static const struct t2bce_core_client_pm_ops t2audio_pm_ops = {
+ .shutdown = t2audio_pm_shutdown_client,
+ .pm_prepare = t2audio_pm_prepare_client,
+};
+
+static int t2audio_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+ struct t2audio_device *t2audio = NULL;
+ struct t2audio_subdevice *sdev = NULL;
+ int status = 0;
+ u32 cfg;
+
+ pr_debug("t2bce_audio: capturing our device\n");
+
+ if (pci_enable_device(dev))
+ return -ENODEV;
+ if (pci_request_regions(dev, "t2bce_audio")) {
+ status = -ENODEV;
+ goto fail;
+ }
+ pci_set_master(dev);
+
+ t2audio = kzalloc(sizeof(struct t2audio_device), GFP_KERNEL);
+ if (!t2audio) {
+ status = -ENOMEM;
+ goto fail;
+ }
+
+ t2audio->bce = t2bce_core_client_get(&dev->dev);
+ if (IS_ERR(t2audio->bce)) {
+ status = PTR_ERR(t2audio->bce);
+ t2audio->bce = NULL;
+ if (status != -EPROBE_DEFER)
+ dev_warn(&dev->dev, "t2bce_audio: Failed to get BCE client: %d\n", status);
+ goto fail;
+ }
+
+ t2audio->pci = dev;
+ pci_set_drvdata(dev, t2audio);
+ t2bce_core_client_set_resume_complete_callback(t2audio->bce, t2audio_resume_complete, t2audio);
+ t2bce_core_client_set_pm_ops(t2audio->bce, &t2audio_pm_ops, t2audio);
+
+ t2audio->devt = t2audio_chrdev;
+ t2audio->dev = device_create(t2audio_class, &dev->dev, t2audio->devt, NULL, "t2bce_audio");
+ if (IS_ERR_OR_NULL(t2audio->dev)) {
+ status = PTR_ERR(t2audio->dev);
+ goto fail;
+ }
+ init_completion(&t2audio->remote_alive);
+ INIT_WORK(&t2audio->resume_work, t2audio_resume_work);
+ INIT_LIST_HEAD(&t2audio->subdevice_list);
+ spin_lock_init(&t2audio->clock_lock);
+ t2audio->clock_samples = 0;
+ t2audio->clock_offset_valid = false;
+
+ /* Init: set an unknown flag in the bitset */
+ if (pci_read_config_dword(dev, 4, &cfg))
+ dev_warn(&dev->dev, "t2bce_audio: pci_read_config_dword fail\n");
+ if (pci_write_config_dword(dev, 4, cfg | 6u))
+ dev_warn(&dev->dev, "t2bce_audio: pci_write_config_dword fail\n");
+
+ pr_debug("t2bce_audio: bs len = %llx\n", pci_resource_len(dev, 0));
+ t2audio->reg_mem_bs_dma = pci_resource_start(dev, 0);
+ t2audio->reg_mem_bs = pci_iomap(dev, 0, 0);
+ t2audio->reg_mem_cfg = pci_iomap(dev, 4, 0);
+
+ t2audio->reg_mem_gpr = (u32 __iomem *) ((u8 __iomem *) t2audio->reg_mem_cfg + 0xC000);
+
+ if (IS_ERR_OR_NULL(t2audio->reg_mem_bs) || IS_ERR_OR_NULL(t2audio->reg_mem_cfg)) {
+ dev_warn(&dev->dev, "t2bce_audio: Failed to pci_iomap required regions\n");
+ goto fail;
+ }
+
+ if (t2audio_bce_init(t2audio)) {
+ dev_warn(&dev->dev, "t2bce_audio: Failed to init BCE command transport\n");
+ goto fail;
+ }
+
+ if (snd_card_new(t2audio->dev, t2audio_alsa_index, t2audio_alsa_id, THIS_MODULE, 0, &t2audio->card)) {
+ dev_err(&dev->dev, "t2bce_audio: Failed to create ALSA card\n");
+ goto fail;
+ }
+
+ strcpy(t2audio->card->shortname, "Apple T2 Audio");
+ strcpy(t2audio->card->longname, "Apple T2 Audio");
+ strcpy(t2audio->card->mixername, "Apple T2 Audio");
+ /* Dynamic alsa ids start at 100 */
+ t2audio->next_alsa_id = 100;
+
+ if (t2audio_init_cmd(t2audio)) {
+ dev_err(&dev->dev, "t2bce_audio: Failed to initialize over BCE\n");
+ goto fail_snd;
+ }
+
+ if (t2audio_init_bs(t2audio)) {
+ dev_err(&dev->dev, "t2bce_audio: Failed to initialize BufferStruct\n");
+ goto fail_snd;
+ }
+
+ if ((status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON))) {
+ dev_err(&dev->dev, "Failed to set remote access\n");
+ goto fail_snd;
+ }
+
+ if (snd_card_register(t2audio->card)) {
+ dev_err(&dev->dev, "t2bce_audio: Failed to register ALSA sound device\n");
+ goto fail_snd;
+ }
+
+ list_for_each_entry(sdev, &t2audio->subdevice_list, list) {
+ struct t2audio_buffer_struct_device *dev = &t2audio->bs->devices[sdev->buf_id];
+
+ if (sdev->out_stream_cnt == 1 && !strcmp(dev->name, "Speaker")) {
+ struct snd_pcm_hardware *hw = sdev->out_streams[0].alsa_hw_desc;
+
+ snprintf(t2audio->card->driver, sizeof(t2audio->card->driver) / sizeof(char), "AppleT2x%d", hw->channels_min);
+ }
+ }
+
+ return 0;
+
+fail_snd:
+ snd_card_free(t2audio->card);
+fail:
+ if (t2audio) {
+ if (t2audio->dev)
+ device_destroy(t2audio_class, t2audio->devt);
+ if (!IS_ERR_OR_NULL(t2audio->reg_mem_bs))
+ pci_iounmap(dev, t2audio->reg_mem_bs);
+ if (!IS_ERR_OR_NULL(t2audio->reg_mem_cfg))
+ pci_iounmap(dev, t2audio->reg_mem_cfg);
+ if (t2audio->bce)
+ t2bce_core_client_set_pm_ops(t2audio->bce, NULL, NULL);
+ t2bce_core_client_put(t2audio->bce);
+ kfree(t2audio);
+ }
+
+ pci_release_regions(dev);
+ pci_disable_device(dev);
+
+ if (!status)
+ status = -EINVAL;
+ return status;
+}
+
+
+
+static void t2audio_remove(struct pci_dev *dev)
+{
+ struct t2audio_subdevice *sdev;
+ struct t2audio_device *t2audio = pci_get_drvdata(dev);
+
+ cancel_work_sync(&t2audio->resume_work);
+ snd_card_free(t2audio->card);
+ while (!list_empty(&t2audio->subdevice_list)) {
+ sdev = list_first_entry(&t2audio->subdevice_list, struct t2audio_subdevice, list);
+ list_del(&sdev->list);
+ t2audio_free_dev(sdev);
+ }
+ pci_iounmap(dev, t2audio->reg_mem_bs);
+ pci_iounmap(dev, t2audio->reg_mem_cfg);
+ device_destroy(t2audio_class, t2audio->devt);
+ pci_free_irq_vectors(dev);
+ pci_release_regions(dev);
+ pci_disable_device(dev);
+ t2bce_core_client_set_pm_ops(t2audio->bce, NULL, NULL);
+ t2bce_core_client_put(t2audio->bce);
+ kfree(t2audio);
+}
+
+static int t2audio_quiesce(struct t2audio_device *t2audio, bool suspend_pcm)
+{
+ struct t2audio_subdevice *sdev;
+ size_t i;
+ int status;
+
+ if (t2audio->pm_quiesced)
+ return 0;
+
+ cancel_work_sync(&t2audio->resume_work);
+ t2audio->resume_deferred = false;
+
+ /* Suspend PCM streams */
+ list_for_each_entry(sdev, &t2audio->subdevice_list, list) {
+ bool stopped_io = false;
+
+ for (i = 0; i < sdev->out_stream_cnt; i++) {
+ if (!smp_load_acquire(&sdev->out_streams[i].started))
+ continue;
+ stopped_io = true;
+ smp_store_release(&sdev->out_streams[i].started, 0);
+ }
+
+ for (i = 0; i < sdev->in_stream_cnt; i++) {
+ if (!smp_load_acquire(&sdev->in_streams[i].started))
+ continue;
+ stopped_io = true;
+ smp_store_release(&sdev->in_streams[i].started, 0);
+ }
+
+ if (stopped_io)
+ t2audio_cmd_stop_io(sdev->a, sdev->dev_id);
+
+ if (suspend_pcm && sdev->pcm)
+ snd_pcm_suspend_all(sdev->pcm);
+ }
+
+ status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_OFF);
+ if (status)
+ dev_warn(t2audio->dev, "Failed to reset remote access\n");
+ else
+ t2audio->pm_quiesced = true;
+
+ return status;
+}
+
+static int t2audio_suspend(struct device *dev)
+{
+ struct t2audio_device *t2audio = pci_get_drvdata(to_pci_dev(dev));
+ int status;
+
+ pr_debug("t2bce_audio: suspend entry\n");
+ status = t2audio_quiesce(t2audio, true);
+ pci_disable_device(t2audio->pci);
+ pr_info("t2bce_audio: suspend exit status=%d\n", status);
+ return 0;
+}
+
+static void t2audio_pm_prepare_client(void *userdata)
+{
+ t2audio_quiesce(userdata, true);
+}
+
+static void t2audio_pm_shutdown_client(void *userdata)
+{
+ t2audio_quiesce(userdata, false);
+}
+
+static void t2audio_shutdown(struct pci_dev *dev)
+{
+ struct t2audio_device *t2audio = pci_get_drvdata(dev);
+
+ if (!t2audio)
+ return;
+
+ /*
+ * Shutdown is not remove: keep allocations intact, but stop active audio
+ * IO and revoke remote access while the BCE command transport is still
+ * alive. The t2bce shutdown path will close the shared bus afterwards.
+ */
+ t2audio_quiesce(t2audio, false);
+ pci_disable_device(t2audio->pci);
+}
+
+static int t2audio_resume(struct device *dev)
+{
+ int status;
+ struct t2audio_device *t2audio = pci_get_drvdata(to_pci_dev(dev));
+ bool no_state_resume = t2bce_core_client_no_state_resume(t2audio->bce);
+ const char *path = no_state_resume ? "no-state" : "stateful";
+
+ if ((status = pci_enable_device(t2audio->pci))) {
+ pr_info("t2bce_audio: resume exit status=%d path=%s\n", status, path);
+ return status;
+ }
+ pci_set_master(t2audio->pci);
+
+ /* we are deferring t2audio resume here until vhci is finished*/
+ if (!no_state_resume) {
+ t2audio->resume_deferred = true;
+ return 0;
+ }
+
+ if ((status = t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON))) {
+ dev_err(t2audio->dev, "Failed to set remote access\n");
+ pr_info("t2bce_audio: resume exit status=%d path=%s\n", status, path);
+ return status;
+ }
+
+ t2audio->resume_deferred = false;
+ t2audio->pm_quiesced = false;
+ t2audio_reset_clock(t2audio);
+ t2audio_reset_streams(t2audio);
+
+ pr_info("t2bce_audio: resume exit status=0 path=%s\n", path);
+ return 0;
+}
+
+static void t2audio_resume_work(struct work_struct *ws)
+{
+ struct t2audio_device *t2audio = container_of(ws, struct t2audio_device, resume_work);
+
+ if (!t2audio->resume_deferred)
+ return;
+
+ if (t2audio_cmd_set_remote_access(t2audio, T2AUDIO_REMOTE_ACCESS_ON)) {
+ t2audio->resume_deferred = false;
+ dev_err(t2audio->dev, "Deferred remote access enable failed\n");
+ pr_info("t2bce_audio: resume deferred path failed\n");
+ return;
+ }
+
+ t2audio->resume_deferred = false;
+ t2audio->pm_quiesced = false;
+ t2audio_reset_clock(t2audio);
+ t2audio_reset_streams(t2audio);
+ pr_info("t2bce_audio: resume deferred path complete\n");
+}
+
+static void t2audio_resume_complete(void *userdata)
+{
+ struct t2audio_device *t2audio = userdata;
+
+ if (!t2audio || !t2audio->resume_deferred)
+ return;
+
+ schedule_work(&t2audio->resume_work);
+}
+
+static void t2audio_reset_stream(struct t2audio_stream *stream)
+{
+ smp_store_release(&stream->started, 0);
+ stream->waiting_for_first_ts = true;
+ stream->remote_timestamp = 0;
+ stream->timestamp_accept_after = 0;
+ stream->clock_offset_ns = 0;
+ stream->timestamp_seed = 0;
+ stream->clock_offset_valid = false;
+ stream->frame_min = stream->latency;
+}
+
+static void t2audio_reset_clock(struct t2audio_device *a)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&a->clock_lock, flags);
+ a->clock_offset_ns = 0;
+ a->clock_samples = 0;
+ a->clock_offset_valid = false;
+ spin_unlock_irqrestore(&a->clock_lock, flags);
+}
+
+static void t2audio_reset_streams(struct t2audio_device *a)
+{
+ struct t2audio_subdevice *sdev;
+ size_t i;
+
+ list_for_each_entry(sdev, &a->subdevice_list, list) {
+ for (i = 0; i < sdev->in_stream_cnt; i++)
+ t2audio_reset_stream(&sdev->in_streams[i]);
+ for (i = 0; i < sdev->out_stream_cnt; i++)
+ t2audio_reset_stream(&sdev->out_streams[i]);
+ }
+}
+
+static int t2audio_init_cmd(struct t2audio_device *a)
+{
+ int status;
+ struct t2audio_send_ctx sctx;
+ struct t2audio_msg buf;
+ u64 dev_cnt, dev_i;
+ t2audio_device_id_t *dev_l;
+
+ if ((status = t2audio_send(a, &sctx, 500,
+ t2audio_msg_write_alive_notification, 1, 3))) {
+ dev_err(a->dev, "Sending alive notification failed\n");
+ return status;
+ }
+
+ if (wait_for_completion_timeout(&a->remote_alive, msecs_to_jiffies(500)) == 0) {
+ dev_err(a->dev, "Timed out waiting for remote\n");
+ return -ETIMEDOUT;
+ }
+ pr_debug("t2bce_audio: Continuing init\n");
+
+ buf = t2audio_reply_alloc();
+ if ((status = t2audio_cmd_get_device_list(a, &buf, &dev_l, &dev_cnt))) {
+ dev_err(a->dev, "Failed to get device list\n");
+ t2audio_reply_free(&buf);
+ return status;
+ }
+ for (dev_i = 0; dev_i < dev_cnt; ++dev_i)
+ t2audio_init_dev(a, dev_l[dev_i]);
+ t2audio_reply_free(&buf);
+
+ return 0;
+}
+
+static void t2audio_init_stream_info(struct t2audio_subdevice *sdev, struct t2audio_stream *strm);
+static void t2audio_handle_jack_connection_change(struct t2audio_subdevice *sdev);
+
+static void t2audio_init_dev(struct t2audio_device *a, t2audio_device_id_t dev_id)
+{
+ struct t2audio_subdevice *sdev;
+ struct t2audio_msg buf = t2audio_reply_alloc();
+ u64 uid_len, stream_cnt, i;
+ t2audio_object_id_t *stream_list;
+ char *uid;
+
+ sdev = kzalloc(sizeof(struct t2audio_subdevice), GFP_KERNEL);
+
+ if (t2audio_cmd_get_property(a, &buf, dev_id, dev_id, T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_UID, 0),
+ NULL, 0, (void **) &uid, &uid_len) || uid_len > T2AUDIO_DEVICE_MAX_UID_LEN) {
+ dev_err(a->dev, "Failed to get device uid for device %llx\n", dev_id);
+ goto fail;
+ }
+ uid[uid_len] = '\0';
+ pr_debug("t2bce_audio: Remote device %llx %.*s\n", dev_id, (int) uid_len, uid);
+
+ sdev->a = a;
+ INIT_LIST_HEAD(&sdev->list);
+ sdev->dev_id = dev_id;
+ sdev->buf_id = T2AUDIO_BUFFER_ID_NONE;
+ strscpy(sdev->uid, uid);
+
+ if (t2audio_cmd_get_primitive_property(a, dev_id, dev_id,
+ T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_INPUT, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->in_latency, sizeof(u32)))
+ dev_warn(a->dev, "Failed to query device input latency\n");
+ if (t2audio_cmd_get_primitive_property(a, dev_id, dev_id,
+ T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &sdev->out_latency, sizeof(u32)))
+ dev_warn(a->dev, "Failed to query device output latency\n");
+
+ if (t2audio_cmd_get_input_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) {
+ dev_err(a->dev, "Failed to get input stream list for device %llx\n", dev_id);
+ goto fail;
+ }
+ if (stream_cnt > T2AUDIO_DEVICE_MAX_INPUT_STREAMS) {
+ dev_warn(a->dev, "Device %s input stream count %llu is larger than the supported count of %u\n",
+ sdev->uid, stream_cnt, T2AUDIO_DEVICE_MAX_INPUT_STREAMS);
+ stream_cnt = T2AUDIO_DEVICE_MAX_INPUT_STREAMS;
+ }
+ sdev->in_stream_cnt = stream_cnt;
+ for (i = 0; i < stream_cnt; i++) {
+ sdev->in_streams[i].id = stream_list[i];
+ sdev->in_streams[i].buffer_cnt = 0;
+ t2audio_init_stream_info(sdev, &sdev->in_streams[i]);
+ sdev->in_streams[i].latency += sdev->in_latency;
+ }
+
+ if (t2audio_cmd_get_output_stream_list(a, &buf, dev_id, &stream_list, &stream_cnt)) {
+ dev_err(a->dev, "Failed to get output stream list for device %llx\n", dev_id);
+ goto fail;
+ }
+ if (stream_cnt > T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS) {
+ dev_warn(a->dev, "Device %s output stream count %llu is larger than the supported count of %u\n",
+ sdev->uid, stream_cnt, T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS);
+ stream_cnt = T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS;
+ }
+ sdev->out_stream_cnt = stream_cnt;
+ for (i = 0; i < stream_cnt; i++) {
+ sdev->out_streams[i].id = stream_list[i];
+ sdev->out_streams[i].buffer_cnt = 0;
+ t2audio_init_stream_info(sdev, &sdev->out_streams[i]);
+ sdev->out_streams[i].latency += sdev->out_latency;
+ }
+
+ if (sdev->is_pcm)
+ t2audio_create_pcm(sdev);
+ /* Headphone Jack status */
+ if (!strcmp(sdev->uid, "Codec Output")) {
+ if (snd_jack_new(a->card, sdev->uid, SND_JACK_HEADPHONE, &sdev->jack, true, false))
+ dev_warn(a->dev, "Failed to create an attached jack for %s\n", sdev->uid);
+ t2audio_cmd_property_listener(a, sdev->dev_id, sdev->dev_id,
+ T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_JACK_PLUGGED, 0));
+ t2audio_handle_jack_connection_change(sdev);
+ }
+
+ t2audio_reply_free(&buf);
+
+ list_add_tail(&sdev->list, &a->subdevice_list);
+ return;
+
+fail:
+ t2audio_reply_free(&buf);
+ kfree(sdev);
+}
+
+static void t2audio_init_stream_info(struct t2audio_subdevice *sdev, struct t2audio_stream *strm)
+{
+ if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id,
+ T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_PHYS_FORMAT, 0), NULL, 0,
+ &strm->desc, sizeof(strm->desc)))
+ dev_warn(sdev->a->dev, "Failed to query stream descriptor\n");
+ if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, strm->id,
+ T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_GLOBAL, T2AUDIO_PROP_LATENCY, 0), NULL, 0, &strm->latency, sizeof(u32)))
+ dev_warn(sdev->a->dev, "Failed to query stream latency\n");
+ if (strm->desc.format_id == T2AUDIO_FORMAT_LPCM)
+ sdev->is_pcm = true;
+}
+
+static void t2audio_free_dev(struct t2audio_subdevice *sdev)
+{
+ size_t i;
+ for (i = 0; i < sdev->in_stream_cnt; i++) {
+ if (sdev->in_streams[i].alsa_hw_desc)
+ kfree(sdev->in_streams[i].alsa_hw_desc);
+ if (sdev->in_streams[i].buffers)
+ kfree(sdev->in_streams[i].buffers);
+ }
+ for (i = 0; i < sdev->out_stream_cnt; i++) {
+ if (sdev->out_streams[i].alsa_hw_desc)
+ kfree(sdev->out_streams[i].alsa_hw_desc);
+ if (sdev->out_streams[i].buffers)
+ kfree(sdev->out_streams[i].buffers);
+ }
+ kfree(sdev);
+}
+
+static struct t2audio_subdevice *t2audio_find_dev_by_dev_id(struct t2audio_device *a, t2audio_device_id_t dev_id)
+{
+ struct t2audio_subdevice *sdev;
+ list_for_each_entry(sdev, &a->subdevice_list, list) {
+ if (dev_id == sdev->dev_id)
+ return sdev;
+ }
+ return NULL;
+}
+
+static struct t2audio_subdevice *t2audio_find_dev_by_uid(struct t2audio_device *a, const char *uid)
+{
+ struct t2audio_subdevice *sdev;
+ list_for_each_entry(sdev, &a->subdevice_list, list) {
+ if (!strcmp(uid, sdev->uid))
+ return sdev;
+ }
+ return NULL;
+}
+
+static void t2audio_init_bs_stream(struct t2audio_device *a, struct t2audio_stream *strm,
+ struct t2audio_buffer_struct_stream *bs_strm);
+static void t2audio_init_bs_stream_host(struct t2audio_device *a, struct t2audio_stream *strm,
+ struct t2audio_buffer_struct_stream *bs_strm);
+
+static int t2audio_init_bs(struct t2audio_device *a)
+{
+ int i, j;
+ struct t2audio_buffer_struct_device *dev;
+ struct t2audio_subdevice *sdev;
+ u32 ver, sig, bs_base;
+
+ ver = ioread32(&a->reg_mem_gpr[0]);
+ if (ver < 3) {
+ dev_err(a->dev, "t2bce_audio: Bad GPR version (%u)", ver);
+ return -EINVAL;
+ }
+ sig = ioread32(&a->reg_mem_gpr[1]);
+ if (sig != T2AUDIO_SIG) {
+ dev_err(a->dev, "t2bce_audio: Bad GPR sig (%x)", sig);
+ return -EINVAL;
+ }
+ bs_base = ioread32(&a->reg_mem_gpr[2]);
+ a->bs = (struct t2audio_buffer_struct *) ((u8 *) a->reg_mem_bs + bs_base);
+ if (a->bs->signature != T2AUDIO_SIG) {
+ dev_err(a->dev, "t2bce_audio: Bad BufferStruct sig (%x)", a->bs->signature);
+ return -EINVAL;
+ }
+ pr_debug("t2bce_audio: BufferStruct ver = %i\n", a->bs->version);
+ pr_debug("t2bce_audio: Num devices = %i\n", a->bs->num_devices);
+ for (i = 0; i < a->bs->num_devices; i++) {
+ dev = &a->bs->devices[i];
+ pr_debug("t2bce_audio: Device %i %s\n", i, dev->name);
+
+ sdev = t2audio_find_dev_by_uid(a, dev->name);
+ if (!sdev) {
+ dev_err(a->dev, "t2bce_audio: Subdevice not found for BufferStruct device %s\n", dev->name);
+ continue;
+ }
+ sdev->buf_id = (u8) i;
+ dev->num_input_streams = 0;
+ for (j = 0; j < dev->num_output_streams; j++) {
+ pr_debug("t2bce_audio: Device %i Stream %i: Output; Buffer Count = %i\n", i, j,
+ dev->output_streams[j].num_buffers);
+ if (j < sdev->out_stream_cnt)
+ t2audio_init_bs_stream(a, &sdev->out_streams[j], &dev->output_streams[j]);
+ }
+ }
+
+ list_for_each_entry(sdev, &a->subdevice_list, list) {
+ if (sdev->buf_id != T2AUDIO_BUFFER_ID_NONE)
+ continue;
+ sdev->buf_id = i;
+ pr_debug("t2bce_audio: Created device %i %s\n", i, sdev->uid);
+ strcpy(a->bs->devices[i].name, sdev->uid);
+ a->bs->devices[i].num_input_streams = 0;
+ a->bs->devices[i].num_output_streams = 0;
+ a->bs->num_devices = ++i;
+ }
+ list_for_each_entry(sdev, &a->subdevice_list, list) {
+ if (sdev->in_stream_cnt == 1) {
+ pr_debug("t2bce_audio: Device %i Host Stream; Input\n", sdev->buf_id);
+ t2audio_init_bs_stream_host(a, &sdev->in_streams[0], &a->bs->devices[sdev->buf_id].input_streams[0]);
+ a->bs->devices[sdev->buf_id].num_input_streams = 1;
+ wmb();
+
+ if (t2audio_cmd_set_input_stream_address_ranges(a, sdev->dev_id)) {
+ dev_err(a->dev, "t2bce_audio: Failed to set input stream address ranges\n");
+ }
+ }
+ }
+
+ return 0;
+}
+
+static void t2audio_init_bs_stream(struct t2audio_device *a, struct t2audio_stream *strm,
+ struct t2audio_buffer_struct_stream *bs_strm)
+{
+ size_t i;
+ strm->buffer_cnt = bs_strm->num_buffers;
+ if (bs_strm->num_buffers > T2AUDIO_DEVICE_MAX_BUFFER_COUNT) {
+ dev_warn(a->dev, "BufferStruct buffer count %u exceeds driver limit of %u\n", bs_strm->num_buffers,
+ T2AUDIO_DEVICE_MAX_BUFFER_COUNT);
+ strm->buffer_cnt = T2AUDIO_DEVICE_MAX_BUFFER_COUNT;
+ }
+ if (!strm->buffer_cnt)
+ return;
+ strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct t2audio_dma_buf), GFP_KERNEL);
+ if (!strm->buffers) {
+ dev_err(a->dev, "Buffer list allocation failed\n");
+ return;
+ }
+ for (i = 0; i < strm->buffer_cnt; i++) {
+ strm->buffers[i].dma_addr = a->reg_mem_bs_dma + (dma_addr_t) bs_strm->buffers[i].address;
+ strm->buffers[i].ptr = a->reg_mem_bs + bs_strm->buffers[i].address;
+ strm->buffers[i].size = bs_strm->buffers[i].size;
+ strm->buffers[i].type = T2AUDIO_DMA_BUF_IOMEM;
+ }
+
+ if (strm->buffer_cnt == 1) {
+ strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL);
+ if (t2audio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) {
+ kfree(strm->alsa_hw_desc);
+ strm->alsa_hw_desc = NULL;
+ }
+ }
+}
+
+static void t2audio_init_bs_stream_host(struct t2audio_device *a, struct t2audio_stream *strm,
+ struct t2audio_buffer_struct_stream *bs_strm)
+{
+ size_t size;
+ dma_addr_t dma_addr;
+ void *dma_ptr;
+ size = strm->desc.bytes_per_packet * 16640;
+ dma_ptr = dma_alloc_coherent(&a->pci->dev, size, &dma_addr, GFP_KERNEL);
+ if (!dma_ptr) {
+ dev_err(a->dev, "dma_alloc_coherent failed\n");
+ return;
+ }
+ bs_strm->buffers[0].address = dma_addr;
+ bs_strm->buffers[0].size = size;
+ bs_strm->num_buffers = 1;
+
+ memset(dma_ptr, 0, size);
+
+ strm->buffer_cnt = 1;
+ strm->buffers = kmalloc_array(strm->buffer_cnt, sizeof(struct t2audio_dma_buf), GFP_KERNEL);
+ if (!strm->buffers) {
+ dev_err(a->dev, "Buffer list allocation failed\n");
+ return;
+ }
+ strm->buffers[0].dma_addr = dma_addr;
+ strm->buffers[0].ptr = dma_ptr;
+ strm->buffers[0].size = size;
+ strm->buffers[0].type = T2AUDIO_DMA_BUF_COHERENT;
+
+ strm->alsa_hw_desc = kmalloc(sizeof(struct snd_pcm_hardware), GFP_KERNEL);
+ if (t2audio_create_hw_info(&strm->desc, strm->alsa_hw_desc, strm->buffers[0].size)) {
+ kfree(strm->alsa_hw_desc);
+ strm->alsa_hw_desc = NULL;
+ }
+}
+
+static void t2audio_handle_prop_change(struct t2audio_device *a, struct t2audio_msg *msg);
+
+void t2audio_handle_notification(struct t2audio_device *a, struct t2audio_msg *msg)
+{
+ struct t2audio_send_ctx sctx;
+ struct t2audio_msg_base base;
+ if (t2audio_msg_read_base(msg, &base))
+ return;
+ switch (base.msg) {
+ case T2AUDIO_MSG_NOTIFICATION_BOOT:
+ pr_debug("t2bce_audio: Received boot notification from remote\n");
+
+ /* Resend the alive notify */
+ if (t2audio_send(a, &sctx, 500,
+ t2audio_msg_write_alive_notification, 1, 3)) {
+ pr_err("Sending alive notification failed\n");
+ }
+ break;
+ case T2AUDIO_MSG_NOTIFICATION_ALIVE:
+ pr_debug("t2bce_audio: Received alive notification from remote\n");
+ complete_all(&a->remote_alive);
+ break;
+ case T2AUDIO_MSG_PROPERTY_CHANGED:
+ t2audio_handle_prop_change(a, msg);
+ break;
+ default:
+ pr_debug("t2bce_audio: Unhandled notification %i\n", base.msg);
+ break;
+ }
+}
+
+struct t2audio_prop_change_work_struct {
+ struct work_struct ws;
+ struct t2audio_device *a;
+ t2audio_device_id_t dev;
+ t2audio_object_id_t obj;
+ struct t2audio_prop_addr prop;
+};
+
+static void t2audio_handle_jack_connection_change(struct t2audio_subdevice *sdev)
+{
+ u32 plugged;
+ if (!sdev->jack)
+ return;
+ /* NOTE: Apple made the plug status scoped to the input and output streams. This makes no sense for us, so I just
+ * always pick the OUTPUT status. */
+ if (t2audio_cmd_get_primitive_property(sdev->a, sdev->dev_id, sdev->dev_id,
+ T2AUDIO_PROP(T2AUDIO_PROP_SCOPE_OUTPUT, T2AUDIO_PROP_JACK_PLUGGED, 0), NULL, 0, &plugged, sizeof(plugged))) {
+ dev_err(sdev->a->dev, "Failed to get jack enable status\n");
+ return;
+ }
+ pr_debug("t2bce_audio: Jack is now %s\n", plugged ? "plugged" : "unplugged");
+ snd_jack_report(sdev->jack, plugged ? sdev->jack->type : 0);
+}
+
+void t2audio_handle_prop_change_work(struct work_struct *ws)
+{
+ struct t2audio_prop_change_work_struct *work = container_of(ws, struct t2audio_prop_change_work_struct, ws);
+ struct t2audio_subdevice *sdev;
+
+ sdev = t2audio_find_dev_by_dev_id(work->a, work->dev);
+ if (!sdev) {
+ dev_err(work->a->dev, "Property notification change: device not found\n");
+ goto done;
+ }
+ pr_debug("t2bce_audio: Property changed for device: %s\n", sdev->uid);
+
+ if (work->prop.scope == T2AUDIO_PROP_SCOPE_OUTPUT && work->prop.selector == T2AUDIO_PROP_JACK_PLUGGED) {
+ t2audio_handle_jack_connection_change(sdev);
+ }
+
+done:
+ kfree(work);
+}
+
+void t2audio_handle_prop_change(struct t2audio_device *a, struct t2audio_msg *msg)
+{
+ /* NOTE: This is a scheduled work because this callback will generally need to query device information and this
+ * is not possible when we are in the reply parsing code's context. */
+ struct t2audio_prop_change_work_struct *work;
+ work = kmalloc(sizeof(struct t2audio_prop_change_work_struct), GFP_KERNEL);
+ work->a = a;
+ INIT_WORK(&work->ws, t2audio_handle_prop_change_work);
+ t2audio_msg_read_property_changed(msg, &work->dev, &work->obj, &work->prop);
+ schedule_work(&work->ws);
+}
+
+#define t2audio_send_cmd_response(a, sctx, msg, fn, ...) \
+ if (t2audio_send_with_tag(a, sctx, ((struct t2audio_msg_header *) msg->data)->tag, 500, fn, ##__VA_ARGS__)) \
+ pr_err("t2bce_audio: Failed to reply to a command\n");
+
+void t2audio_handle_cmd_timestamp(struct t2audio_device *a, struct t2audio_msg *msg)
+{
+ ktime_t time_os = ktime_get_boottime();
+ unsigned long flags;
+ struct t2audio_send_ctx sctx;
+ struct t2audio_subdevice *sdev;
+ bool clock_ready;
+ s64 clock_offset_ns;
+ s64 offset_sample;
+ u64 devid, timestamp, update_seed;
+ t2audio_msg_read_update_timestamp(msg, &devid, &timestamp, &update_seed);
+
+ offset_sample = ktime_to_ns(time_os) - (s64)timestamp;
+ spin_lock_irqsave(&a->clock_lock, flags);
+ if (offset_sample >= 0) {
+ if (!a->clock_offset_valid || offset_sample < a->clock_offset_ns) {
+ a->clock_offset_ns = offset_sample;
+ a->clock_offset_valid = true;
+ }
+ if (a->clock_samples < 2)
+ a->clock_samples++;
+ }
+ clock_offset_ns = a->clock_offset_ns;
+ clock_ready = a->clock_samples >= 2;
+ spin_unlock_irqrestore(&a->clock_lock, flags);
+ pr_debug("t2bce_audio: timestamp dev=%llx t2=%llx host=%lld seed=%llx sample_offset=%lld clock_offset=%lld\n",
+ devid, timestamp, ktime_to_ns(time_os), update_seed,
+ offset_sample, clock_offset_ns);
+
+ sdev = t2audio_find_dev_by_dev_id(a, devid);
+ if (sdev)
+ t2audio_handle_timestamp(sdev, timestamp, update_seed, clock_offset_ns,
+ clock_ready);
+
+ t2audio_send_cmd_response(a, &sctx, msg,
+ t2audio_msg_write_update_timestamp_response);
+}
+
+void t2audio_handle_command(struct t2audio_device *a, struct t2audio_msg *msg)
+{
+ struct t2audio_msg_base base;
+ if (t2audio_msg_read_base(msg, &base))
+ return;
+ switch (base.msg) {
+ case T2AUDIO_MSG_UPDATE_TIMESTAMP:
+ t2audio_handle_cmd_timestamp(a, msg);
+ break;
+ default:
+ pr_debug("t2bce_audio: Unhandled device command %i\n", base.msg);
+ break;
+ }
+}
+
+static struct pci_device_id t2audio_ids[ ] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1803) },
+ { 0, },
+};
+MODULE_DEVICE_TABLE(pci, t2audio_ids);
+
+struct dev_pm_ops t2audio_pci_driver_pm = {
+ .suspend = t2audio_suspend,
+ .resume = t2audio_resume
+};
+struct pci_driver t2audio_pci_driver = {
+ .name = "t2bce_audio",
+ .id_table = t2audio_ids,
+ .probe = t2audio_probe,
+ .remove = t2audio_remove,
+ .shutdown = t2audio_shutdown,
+ .driver = {
+ .pm = &t2audio_pci_driver_pm
+ }
+};
+
+
+static int __init t2audio_module_init(void)
+{
+ int result;
+ if ((result = alloc_chrdev_region(&t2audio_chrdev, 0, 1, "t2bce_audio")))
+ goto fail_chrdev;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0)
+ t2audio_class = class_create(THIS_MODULE, "t2bce_audio");
+#else
+ t2audio_class = class_create("t2bce_audio");
+#endif
+ if (IS_ERR(t2audio_class)) {
+ result = PTR_ERR(t2audio_class);
+ goto fail_class;
+ }
+
+ result = pci_register_driver(&t2audio_pci_driver);
+ if (result)
+ goto fail_drv;
+ pr_info("t2bce_audio: module initialized\n");
+ return 0;
+
+fail_drv:
+ pci_unregister_driver(&t2audio_pci_driver);
+fail_class:
+ class_destroy(t2audio_class);
+fail_chrdev:
+ unregister_chrdev_region(t2audio_chrdev, 1);
+ if (!result)
+ result = -EINVAL;
+ pr_info("t2bce_audio: module init failed status=%d\n", result);
+ return result;
+}
+
+static void __exit t2audio_module_exit(void)
+{
+ pci_unregister_driver(&t2audio_pci_driver);
+ class_destroy(t2audio_class);
+ unregister_chrdev_region(t2audio_chrdev, 1);
+ pr_info("t2bce_audio: module exited\n");
+}
+
+struct t2audio_alsa_pcm_id_mapping t2audio_alsa_id_mappings[] = {
+ {"Speaker", 0},
+ {"Digital Mic", 1},
+ {"Codec Output", 2},
+ {"Codec Input", 3},
+ {"Bridge Loopback", 4},
+ {}
+};
+
+module_param_named(index, t2audio_alsa_index, int, 0444);
+MODULE_PARM_DESC(index, "Index value for Apple Internal Audio soundcard.");
+module_param_named(id, t2audio_alsa_id, charp, 0444);
+MODULE_PARM_DESC(id, "ID string for Apple Internal Audio soundcard.");
+MODULE_SOFTDEP("pre: t2bce_core");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("André Eikmeyer <andre.eikmeyer@gmail.com>");
+MODULE_DESCRIPTION("Apple T2 Audio Driver");
+MODULE_VERSION("0.01");
+module_init(t2audio_module_init);
+module_exit(t2audio_module_exit);
diff --git a/drivers/staging/t2bce/t2bce_audio/audio.h b/drivers/staging/t2bce/t2bce_audio/audio.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/audio.h
@@ -0,0 +1,148 @@
+#ifndef T2AUDIO_H
+#define T2AUDIO_H
+
+#include <linux/types.h>
+#include <linux/workqueue.h>
+#include <sound/pcm.h>
+#include "t2bce_core_transport.h"
+#include "protocol_bce.h"
+#include "description.h"
+
+#define T2AUDIO_SIG 0x19870423
+
+#define T2AUDIO_DEVICE_MAX_UID_LEN 128
+#define T2AUDIO_DEVICE_MAX_INPUT_STREAMS 1
+#define T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS 1
+#define T2AUDIO_DEVICE_MAX_BUFFER_COUNT 1
+
+#define T2AUDIO_BUFFER_ID_NONE 0xffu
+
+struct snd_card;
+struct snd_pcm;
+struct snd_pcm_hardware;
+struct snd_jack;
+
+struct __attribute__((packed)) __attribute__((aligned(4))) t2audio_buffer_struct_buffer {
+ size_t address;
+ size_t size;
+ size_t pad[4];
+};
+struct t2audio_buffer_struct_stream {
+ u8 num_buffers;
+ struct t2audio_buffer_struct_buffer buffers[100];
+ char filler[32];
+};
+struct t2audio_buffer_struct_device {
+ char name[128];
+ u8 num_input_streams;
+ u8 num_output_streams;
+ struct t2audio_buffer_struct_stream input_streams[5];
+ struct t2audio_buffer_struct_stream output_streams[5];
+ char filler[128];
+};
+struct t2audio_buffer_struct {
+ u32 version;
+ u32 signature;
+ u32 flags;
+ u8 num_devices;
+ struct t2audio_buffer_struct_device devices[20];
+};
+
+struct t2audio_device;
+
+struct t2audio_deferred_msg {
+ struct work_struct ws;
+ struct t2audio_device *a;
+ struct t2audio_msg msg;
+};
+
+enum t2audio_dma_buf_type {
+ T2AUDIO_DMA_BUF_IOMEM,
+ T2AUDIO_DMA_BUF_COHERENT,
+};
+
+struct t2audio_dma_buf {
+ dma_addr_t dma_addr;
+ void *ptr;
+ size_t size;
+ enum t2audio_dma_buf_type type;
+};
+struct t2audio_stream {
+ t2audio_object_id_t id;
+ size_t buffer_cnt;
+ struct t2audio_dma_buf *buffers;
+
+ struct t2audio_apple_description desc;
+ struct snd_pcm_hardware *alsa_hw_desc;
+ u32 latency;
+
+ bool waiting_for_first_ts;
+
+ ktime_t remote_timestamp;
+ ktime_t timestamp_accept_after;
+ s64 clock_offset_ns;
+ u64 timestamp_seed;
+ bool clock_offset_valid;
+ snd_pcm_sframes_t frame_min;
+ int started;
+};
+struct t2audio_subdevice {
+ struct t2audio_device *a;
+ struct list_head list;
+ t2audio_device_id_t dev_id;
+ u32 in_latency, out_latency;
+ u8 buf_id;
+ int alsa_id;
+ char uid[T2AUDIO_DEVICE_MAX_UID_LEN + 1];
+ size_t in_stream_cnt;
+ struct t2audio_stream in_streams[T2AUDIO_DEVICE_MAX_INPUT_STREAMS];
+ size_t out_stream_cnt;
+ struct t2audio_stream out_streams[T2AUDIO_DEVICE_MAX_OUTPUT_STREAMS];
+ bool is_pcm;
+ struct snd_pcm *pcm;
+ struct snd_jack *jack;
+};
+struct t2audio_alsa_pcm_id_mapping {
+ const char *name;
+ int alsa_id;
+};
+
+struct t2audio_device {
+ struct pci_dev *pci;
+ dev_t devt;
+ struct device *dev;
+ void __iomem *reg_mem_bs;
+ dma_addr_t reg_mem_bs_dma;
+ void __iomem *reg_mem_cfg;
+
+ u32 __iomem *reg_mem_gpr;
+
+ struct t2audio_buffer_struct *bs;
+
+ struct t2bce_core_client *bce;
+ struct t2audio_bce bcem;
+
+ struct snd_card *card;
+
+ struct list_head subdevice_list;
+ int next_alsa_id;
+
+ struct completion remote_alive;
+ struct work_struct resume_work;
+ bool resume_deferred;
+ bool pm_quiesced;
+
+ spinlock_t clock_lock;
+ s64 clock_offset_ns;
+ u32 clock_samples;
+ bool clock_offset_valid;
+};
+
+void t2audio_handle_notification(struct t2audio_device *a, struct t2audio_msg *msg);
+void t2audio_handle_prop_change_work(struct work_struct *ws);
+void t2audio_handle_cmd_timestamp(struct t2audio_device *a, struct t2audio_msg *msg);
+void t2audio_handle_command(struct t2audio_device *a, struct t2audio_msg *msg);
+
+extern struct t2audio_alsa_pcm_id_mapping t2audio_alsa_id_mappings[];
+
+#endif //T2AUDIO_H
diff --git a/drivers/staging/t2bce/t2bce_audio/description.h b/drivers/staging/t2bce/t2bce_audio/description.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/description.h
@@ -0,0 +1,42 @@
+#ifndef T2AUDIO_DESCRIPTION_H
+#define T2AUDIO_DESCRIPTION_H
+
+#include <linux/types.h>
+
+struct t2audio_apple_description {
+ u64 sample_rate_double;
+ u32 format_id;
+ u32 format_flags;
+ u32 bytes_per_packet;
+ u32 frames_per_packet;
+ u32 bytes_per_frame;
+ u32 channels_per_frame;
+ u32 bits_per_channel;
+ u32 reserved;
+};
+
+enum {
+ T2AUDIO_FORMAT_LPCM = 0x6c70636d // 'lpcm'
+};
+
+enum {
+ T2AUDIO_FORMAT_FLAG_FLOAT = 1,
+ T2AUDIO_FORMAT_FLAG_BIG_ENDIAN = 2,
+ T2AUDIO_FORMAT_FLAG_SIGNED = 4,
+ T2AUDIO_FORMAT_FLAG_PACKED = 8,
+ T2AUDIO_FORMAT_FLAG_ALIGNED_HIGH = 16,
+ T2AUDIO_FORMAT_FLAG_NON_INTERLEAVED = 32,
+ T2AUDIO_FORMAT_FLAG_NON_MIXABLE = 64
+};
+
+static inline u64 t2audio_double_to_u64(u64 d)
+{
+ u8 sign = (u8) ((d >> 63) & 1);
+ s32 exp = (s32) ((d >> 52) & 0x7ff) - 1023;
+ u64 fr = d & ((1LL << 52) - 1);
+ if (sign || exp < 0)
+ return 0;
+ return (u64) ((1LL << exp) + (fr >> (52 - exp)));
+}
+
+#endif //T2AUDIO_DESCRIPTION_H
diff --git a/drivers/staging/t2bce/t2bce_audio/pcm.c b/drivers/staging/t2bce/t2bce_audio/pcm.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/pcm.c
@@ -0,0 +1,520 @@
+#include "pcm.h"
+#include "audio.h"
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/ktime.h>
+
+static u64 t2audio_get_alsa_fmtbit(struct t2audio_apple_description *desc)
+{
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_FLOAT) {
+ if (desc->bits_per_channel == 32) {
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN)
+ return SNDRV_PCM_FMTBIT_FLOAT_BE;
+ else
+ return SNDRV_PCM_FMTBIT_FLOAT_LE;
+ } else if (desc->bits_per_channel == 64) {
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN)
+ return SNDRV_PCM_FMTBIT_FLOAT64_BE;
+ else
+ return SNDRV_PCM_FMTBIT_FLOAT64_LE;
+ } else {
+ pr_err("t2bce_audio: unsupported bits per channel for float format: %u\n", desc->bits_per_channel);
+ return 0;
+ }
+ }
+#define DEFINE_BPC_OPTION(val, b) \
+ case val: \
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_BIG_ENDIAN) { \
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) \
+ return SNDRV_PCM_FMTBIT_S ## b ## BE; \
+ else \
+ return SNDRV_PCM_FMTBIT_U ## b ## BE; \
+ } else { \
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED) \
+ return SNDRV_PCM_FMTBIT_S ## b ## LE; \
+ else \
+ return SNDRV_PCM_FMTBIT_U ## b ## LE; \
+ }
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_PACKED) {
+ switch (desc->bits_per_channel) {
+ case 8:
+ case 16:
+ case 32:
+ break;
+ DEFINE_BPC_OPTION(24, 24_3)
+ default:
+ pr_err("t2bce_audio: unsupported bits per channel for packed format: %u\n", desc->bits_per_channel);
+ return 0;
+ }
+ }
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_ALIGNED_HIGH) {
+ switch (desc->bits_per_channel) {
+ DEFINE_BPC_OPTION(24, 32_)
+ default:
+ pr_err("t2bce_audio: unsupported bits per channel for high-aligned format: %u\n", desc->bits_per_channel);
+ return 0;
+ }
+ }
+ switch (desc->bits_per_channel) {
+ case 8:
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_SIGNED)
+ return SNDRV_PCM_FMTBIT_S8;
+ else
+ return SNDRV_PCM_FMTBIT_U8;
+ DEFINE_BPC_OPTION(16, 16_)
+ DEFINE_BPC_OPTION(24, 24_)
+ DEFINE_BPC_OPTION(32, 32_)
+ default:
+ pr_err("t2bce_audio: unsupported bits per channel: %u\n", desc->bits_per_channel);
+ return 0;
+ }
+}
+int t2audio_create_hw_info(struct t2audio_apple_description *desc, struct snd_pcm_hardware *alsa_hw,
+ size_t buf_size)
+{
+ uint rate;
+ alsa_hw->info = (SNDRV_PCM_INFO_MMAP |
+ SNDRV_PCM_INFO_BLOCK_TRANSFER |
+ SNDRV_PCM_INFO_MMAP_VALID |
+ SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
+ SNDRV_PCM_INFO_DOUBLE);
+ if (desc->format_flags & T2AUDIO_FORMAT_FLAG_NON_MIXABLE)
+ pr_warn("t2bce_audio: unsupported hw flag: NON_MIXABLE\n");
+ if (!(desc->format_flags & T2AUDIO_FORMAT_FLAG_NON_INTERLEAVED))
+ alsa_hw->info |= SNDRV_PCM_INFO_INTERLEAVED;
+ alsa_hw->formats = t2audio_get_alsa_fmtbit(desc);
+ if (!alsa_hw->formats)
+ return -EINVAL;
+ rate = (uint) t2audio_double_to_u64(desc->sample_rate_double);
+ alsa_hw->rates = snd_pcm_rate_to_rate_bit(rate);
+ alsa_hw->rate_min = rate;
+ alsa_hw->rate_max = rate;
+ alsa_hw->channels_min = desc->channels_per_frame;
+ alsa_hw->channels_max = desc->channels_per_frame;
+ alsa_hw->buffer_bytes_max = buf_size;
+ alsa_hw->period_bytes_min = desc->bytes_per_packet;
+ alsa_hw->period_bytes_max = desc->bytes_per_packet;
+ alsa_hw->periods_min = (uint) (buf_size / desc->bytes_per_packet);
+ alsa_hw->periods_max = (uint) (buf_size / desc->bytes_per_packet);
+ pr_debug("t2audio_create_hw_info: format = %llu, rate = %u/%u. channels = %u, periods = %u, period size = %lu\n",
+ alsa_hw->formats, alsa_hw->rate_min, alsa_hw->rates, alsa_hw->channels_min, alsa_hw->periods_min,
+ alsa_hw->period_bytes_min);
+ return 0;
+}
+
+static struct t2audio_stream *t2audio_pcm_stream(struct snd_pcm_substream *substream)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+ return &sdev->out_streams[substream->number];
+ else
+ return &sdev->in_streams[substream->number];
+}
+
+static struct t2audio_dma_buf *t2audio_pcm_dma_buf(struct t2audio_stream *stream)
+{
+ if (!stream->buffer_cnt || !stream->buffers)
+ return NULL;
+
+ return &stream->buffers[0];
+}
+
+static void t2audio_dma_memset(struct t2audio_dma_buf *buf, size_t offset, int value, size_t size)
+{
+ if (!buf || offset >= buf->size)
+ return;
+
+ size = min(size, buf->size - offset);
+ switch (buf->type) {
+ case T2AUDIO_DMA_BUF_IOMEM:
+ memset_io((u8 __iomem *) buf->ptr + offset, value, size);
+ break;
+ case T2AUDIO_DMA_BUF_COHERENT:
+ memset((u8 *) buf->ptr + offset, value, size);
+ break;
+ }
+}
+
+static void t2audio_dma_copy_from(struct t2audio_dma_buf *buf, void *dst, size_t offset, size_t size)
+{
+ if (!buf || offset >= buf->size)
+ return;
+
+ size = min(size, buf->size - offset);
+ switch (buf->type) {
+ case T2AUDIO_DMA_BUF_IOMEM:
+ memcpy_fromio(dst, (u8 __iomem *) buf->ptr + offset, size);
+ break;
+ case T2AUDIO_DMA_BUF_COHERENT:
+ memcpy(dst, (u8 *) buf->ptr + offset, size);
+ break;
+ }
+}
+
+static void t2audio_dma_copy_to(struct t2audio_dma_buf *buf, size_t offset, const void *src, size_t size)
+{
+ if (!buf || offset >= buf->size)
+ return;
+
+ size = min(size, buf->size - offset);
+ switch (buf->type) {
+ case T2AUDIO_DMA_BUF_IOMEM:
+ memcpy_toio((u8 __iomem *) buf->ptr + offset, src, size);
+ break;
+ case T2AUDIO_DMA_BUF_COHERENT:
+ memcpy((u8 *) buf->ptr + offset, src, size);
+ break;
+ }
+}
+
+static void t2audio_pcm_zero_frames(struct snd_pcm_substream *substream,
+ snd_pcm_uframes_t first, snd_pcm_uframes_t frames)
+{
+ struct t2audio_stream *stream = t2audio_pcm_stream(substream);
+ struct t2audio_dma_buf *buf = t2audio_pcm_dma_buf(stream);
+ struct snd_pcm_runtime *runtime = substream->runtime;
+ size_t offset;
+ size_t size;
+
+ if (!buf || !runtime || !runtime->buffer_size || !frames)
+ return;
+
+ first %= runtime->buffer_size;
+ if (frames > runtime->buffer_size)
+ frames = runtime->buffer_size;
+
+ offset = frames_to_bytes(runtime, first);
+ if (first + frames <= runtime->buffer_size) {
+ size = frames_to_bytes(runtime, frames);
+ t2audio_dma_memset(buf, offset, 0, size);
+ return;
+ }
+
+ size = frames_to_bytes(runtime, runtime->buffer_size - first);
+ t2audio_dma_memset(buf, offset, 0, size);
+ t2audio_dma_memset(buf, 0, 0, frames_to_bytes(runtime, frames - (runtime->buffer_size - first)));
+}
+
+static int t2audio_pcm_open(struct snd_pcm_substream *substream)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+
+ pr_debug("t2bce_audio: pcm open dev=%s direction=%s substream=%u\n",
+ sdev->uid,
+ substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture",
+ substream->number);
+ substream->runtime->hw = *t2audio_pcm_stream(substream)->alsa_hw_desc;
+
+ return 0;
+}
+
+static int t2audio_pcm_close(struct snd_pcm_substream *substream)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+
+ pr_debug("t2bce_audio: pcm close dev=%s direction=%s substream=%u\n",
+ sdev->uid,
+ substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture",
+ substream->number);
+ return 0;
+}
+
+static int t2audio_pcm_prepare(struct snd_pcm_substream *substream)
+{
+ struct t2audio_stream *stream = t2audio_pcm_stream(substream);
+
+ stream->waiting_for_first_ts = true;
+ stream->remote_timestamp = 0;
+ stream->timestamp_accept_after = 0;
+ stream->clock_offset_ns = 0;
+ stream->timestamp_seed = 0;
+ stream->clock_offset_valid = false;
+ stream->frame_min = stream->latency;
+
+ if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && substream->runtime->buffer_size)
+ t2audio_pcm_zero_frames(substream, 0, substream->runtime->buffer_size);
+
+ return 0;
+}
+
+static int t2audio_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+ struct t2audio_stream *astream = t2audio_pcm_stream(substream);
+
+ pr_debug("t2bce_audio: pcm hw_params dev=%s direction=%s substream=%u\n",
+ sdev->uid,
+ substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture",
+ substream->number);
+
+ if (!astream->buffer_cnt || !astream->buffers)
+ return -EINVAL;
+
+ substream->runtime->dma_area = astream->buffers[0].ptr;
+ substream->runtime->dma_addr = astream->buffers[0].dma_addr;
+ substream->runtime->dma_bytes = astream->buffers[0].size;
+ return 0;
+}
+
+static int t2audio_pcm_hw_free(struct snd_pcm_substream *substream)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+
+ pr_debug("t2bce_audio: pcm hw_free dev=%s direction=%s substream=%u\n",
+ sdev->uid,
+ substream->stream == SNDRV_PCM_STREAM_PLAYBACK ? "playback" : "capture",
+ substream->number);
+ return 0;
+}
+
+static int t2audio_pcm_start(struct snd_pcm_substream *substream)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+ struct t2audio_stream *stream = t2audio_pcm_stream(substream);
+ struct t2audio_dma_buf *dmabuf = t2audio_pcm_dma_buf(stream);
+ void *buf = NULL;
+ size_t s = 0;
+ ktime_t time_start, time_end;
+ bool back_buffer;
+ int status;
+
+ time_start = ktime_get();
+ smp_store_release(&stream->started, 0);
+
+ back_buffer = (substream->stream == SNDRV_PCM_STREAM_PLAYBACK);
+
+ if (back_buffer) {
+ snd_pcm_uframes_t appl_ptr;
+
+ if (!dmabuf)
+ return -EINVAL;
+ if (!substream->runtime->buffer_size)
+ return -EINVAL;
+
+ appl_ptr = substream->runtime->control->appl_ptr % substream->runtime->buffer_size;
+ s = frames_to_bytes(substream->runtime, appl_ptr);
+ if (s) {
+ buf = kmalloc(s, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ t2audio_dma_copy_from(dmabuf, buf, 0, s);
+ time_end = ktime_get();
+ pr_debug("t2bce_audio: Backed up the buffer in %lluns [%li]\n", ktime_to_ns(time_end - time_start),
+ appl_ptr);
+ }
+ }
+
+ status = t2audio_cmd_start_io(sdev->a, sdev->dev_id);
+ if (back_buffer && buf)
+ t2audio_dma_copy_to(dmabuf, 0, buf, s);
+ kfree(buf);
+
+ if (status)
+ return status;
+
+ /* Timestamps received while START_IO was pending belong to the old epoch. */
+ stream->remote_timestamp = 0;
+ stream->waiting_for_first_ts = true;
+ stream->timestamp_accept_after = ktime_get_boottime();
+ stream->clock_offset_ns = 0;
+ stream->timestamp_seed = 0;
+ stream->clock_offset_valid = false;
+ stream->frame_min = stream->latency;
+ smp_store_release(&stream->started, 1);
+
+ time_end = ktime_get();
+ pr_debug("t2bce_audio: start_io %s %lld us\n",
+ sdev->uid, ktime_to_us(ktime_sub(time_end, time_start)));
+ return 0;
+}
+
+static int t2audio_pcm_trigger(struct snd_pcm_substream *substream, int cmd)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+ struct t2audio_stream *stream = t2audio_pcm_stream(substream);
+ int err;
+
+ /* bridgeOS exposes one ALSA substream per remote stream. */
+ if (substream->number != 0)
+ return 0;
+ switch (cmd) {
+ case SNDRV_PCM_TRIGGER_START:
+ pr_debug("t2bce_audio: TRIGGER START %s\n", sdev->uid);
+ err = t2audio_pcm_start(substream);
+ if (err)
+ return err;
+ break;
+ case SNDRV_PCM_TRIGGER_STOP:
+ pr_debug("t2bce_audio: TRIGGER STOP %s\n", sdev->uid);
+ smp_store_release(&stream->started, 0);
+ err = t2audio_cmd_stop_io(sdev->a, sdev->dev_id);
+ stream->remote_timestamp = 0;
+ stream->waiting_for_first_ts = true;
+ stream->timestamp_accept_after = 0;
+ stream->clock_offset_ns = 0;
+ stream->timestamp_seed = 0;
+ stream->clock_offset_valid = false;
+ if (err)
+ return err;
+ break;
+ default:
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static snd_pcm_uframes_t t2audio_pcm_pointer(struct snd_pcm_substream *substream)
+{
+ struct t2audio_stream *stream = t2audio_pcm_stream(substream);
+ ktime_t time_from_start;
+ snd_pcm_sframes_t frames;
+ snd_pcm_sframes_t buffer_time_length;
+
+ if (!smp_load_acquire(&stream->started) || stream->waiting_for_first_ts)
+ return 0;
+
+ /* bridgeOS reports coarse timestamps; interpolate the ALSA pointer locally. */
+ time_from_start = ktime_get_boottime() - stream->remote_timestamp;
+ if (ktime_to_ns(time_from_start) < 0)
+ return 0;
+ buffer_time_length = NSEC_PER_SEC * substream->runtime->buffer_size / substream->runtime->rate;
+ frames = (ktime_to_ns(time_from_start) % buffer_time_length) * substream->runtime->buffer_size / buffer_time_length;
+ if (ktime_to_ns(time_from_start) < buffer_time_length) {
+ if (frames < stream->frame_min)
+ frames = stream->frame_min;
+ else
+ stream->frame_min = 0;
+ } else {
+ if (ktime_to_ns(time_from_start) < 2 * buffer_time_length)
+ stream->frame_min = frames;
+ else
+ stream->frame_min = 0;
+ }
+ frames -= stream->latency;
+ if (frames < 0)
+ frames += ((-frames - 1) / substream->runtime->buffer_size + 1) * substream->runtime->buffer_size;
+ frames %= substream->runtime->buffer_size;
+ return (snd_pcm_uframes_t) frames;
+}
+
+static int t2audio_pcm_mmap(struct snd_pcm_substream *substream, struct vm_area_struct *area)
+{
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+ struct t2audio_stream *stream = t2audio_pcm_stream(substream);
+ struct t2audio_dma_buf *buf;
+
+ if (!stream->buffer_cnt || !stream->buffers)
+ return -EINVAL;
+
+ buf = &stream->buffers[0];
+ switch (buf->type) {
+ case T2AUDIO_DMA_BUF_IOMEM:
+ return snd_pcm_lib_mmap_iomem(substream, area);
+ case T2AUDIO_DMA_BUF_COHERENT:
+ return dma_mmap_coherent(sdev->a->dev, area, buf->ptr, buf->dma_addr, buf->size);
+ default:
+ return -EINVAL;
+ }
+}
+
+static struct snd_pcm_ops t2audio_pcm_ops = {
+ .open = t2audio_pcm_open,
+ .close = t2audio_pcm_close,
+ .ioctl = snd_pcm_lib_ioctl,
+ .hw_params = t2audio_pcm_hw_params,
+ .hw_free = t2audio_pcm_hw_free,
+ .prepare = t2audio_pcm_prepare,
+ .trigger = t2audio_pcm_trigger,
+ .pointer = t2audio_pcm_pointer,
+ .mmap = t2audio_pcm_mmap
+};
+
+int t2audio_create_pcm(struct t2audio_subdevice *sdev)
+{
+ struct snd_pcm *pcm;
+ struct t2audio_alsa_pcm_id_mapping *id_mapping;
+ int err;
+
+ if (!sdev->is_pcm || (sdev->in_stream_cnt == 0 && sdev->out_stream_cnt == 0)) {
+ return -EINVAL;
+ }
+
+ for (id_mapping = t2audio_alsa_id_mappings; id_mapping->name; id_mapping++) {
+ if (!strcmp(sdev->uid, id_mapping->name)) {
+ sdev->alsa_id = id_mapping->alsa_id;
+ break;
+ }
+ }
+ if (!id_mapping->name)
+ sdev->alsa_id = sdev->a->next_alsa_id++;
+ err = snd_pcm_new(sdev->a->card, sdev->uid, sdev->alsa_id,
+ (int) sdev->out_stream_cnt, (int) sdev->in_stream_cnt, &pcm);
+ if (err < 0)
+ return err;
+ pcm->private_data = sdev;
+ pcm->nonatomic = 1;
+ sdev->pcm = pcm;
+ strcpy(pcm->name, sdev->uid);
+ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &t2audio_pcm_ops);
+ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &t2audio_pcm_ops);
+ return 0;
+}
+
+static void t2audio_handle_stream_timestamp(struct snd_pcm_substream *substream,
+ u64 dev_timestamp, u64 update_seed,
+ s64 clock_offset_ns, bool clock_ready)
+{
+ unsigned long flags;
+ ktime_t event_timestamp;
+ struct t2audio_stream *stream;
+ struct t2audio_subdevice *sdev = snd_pcm_substream_chip(substream);
+
+ stream = t2audio_pcm_stream(substream);
+ snd_pcm_stream_lock_irqsave(substream, flags);
+ if (!smp_load_acquire(&stream->started)) {
+ snd_pcm_stream_unlock_irqrestore(substream, flags);
+ return;
+ }
+ if (!clock_ready) {
+ snd_pcm_stream_unlock_irqrestore(substream, flags);
+ return;
+ }
+ if (!stream->clock_offset_valid || stream->timestamp_seed != update_seed) {
+ stream->clock_offset_ns = clock_offset_ns;
+ stream->timestamp_seed = update_seed;
+ stream->clock_offset_valid = true;
+ }
+ event_timestamp = ns_to_ktime((s64)dev_timestamp + stream->clock_offset_ns);
+ if (ktime_before(event_timestamp, stream->timestamp_accept_after)) {
+ pr_debug("t2bce_audio: ignoring pre-start timestamp dev=%s event=%lld accept_after=%lld\n",
+ sdev->uid, ktime_to_ns(event_timestamp),
+ ktime_to_ns(stream->timestamp_accept_after));
+ snd_pcm_stream_unlock_irqrestore(substream, flags);
+ return;
+ }
+ stream->remote_timestamp = event_timestamp;
+ if (stream->waiting_for_first_ts) {
+ stream->waiting_for_first_ts = false;
+ snd_pcm_stream_unlock_irqrestore(substream, flags);
+ return;
+ }
+ snd_pcm_stream_unlock_irqrestore(substream, flags);
+ if (!substream->runtime->no_period_wakeup)
+ snd_pcm_period_elapsed(substream);
+}
+
+void t2audio_handle_timestamp(struct t2audio_subdevice *sdev, u64 dev_timestamp,
+ u64 update_seed, s64 clock_offset_ns, bool clock_ready)
+{
+ struct snd_pcm_substream *substream;
+
+ substream = sdev->pcm->streams[SNDRV_PCM_STREAM_PLAYBACK].substream;
+ if (substream)
+ t2audio_handle_stream_timestamp(substream, dev_timestamp, update_seed,
+ clock_offset_ns, clock_ready);
+ substream = sdev->pcm->streams[SNDRV_PCM_STREAM_CAPTURE].substream;
+ if (substream)
+ t2audio_handle_stream_timestamp(substream, dev_timestamp, update_seed,
+ clock_offset_ns, clock_ready);
+}
diff --git a/drivers/staging/t2bce/t2bce_audio/pcm.h b/drivers/staging/t2bce/t2bce_audio/pcm.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/pcm.h
@@ -0,0 +1,17 @@
+#ifndef T2AUDIO_PCM_H
+#define T2AUDIO_PCM_H
+
+#include <linux/types.h>
+#include <linux/ktime.h>
+
+struct t2audio_subdevice;
+struct t2audio_apple_description;
+struct snd_pcm_hardware;
+
+int t2audio_create_hw_info(struct t2audio_apple_description *desc, struct snd_pcm_hardware *alsa_hw, size_t buf_size);
+int t2audio_create_pcm(struct t2audio_subdevice *sdev);
+
+void t2audio_handle_timestamp(struct t2audio_subdevice *sdev, u64 dev_timestamp,
+ u64 update_seed, s64 clock_offset_ns, bool clock_ready);
+
+#endif //T2AUDIO_PCM_H
diff --git a/drivers/staging/t2bce/t2bce_audio/protocol.c b/drivers/staging/t2bce/t2bce_audio/protocol.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/protocol.c
@@ -0,0 +1,347 @@
+#include "protocol.h"
+#include "protocol_bce.h"
+#include "audio.h"
+
+int t2audio_msg_read_base(struct t2audio_msg *msg, struct t2audio_msg_base *base)
+{
+ if (msg->size < sizeof(struct t2audio_msg_header) + sizeof(struct t2audio_msg_base) * 2)
+ return -EINVAL;
+ *base = *((struct t2audio_msg_base *) ((struct t2audio_msg_header *) msg->data + 1));
+ return 0;
+}
+
+#define READ_START(type) \
+ size_t offset = sizeof(struct t2audio_msg_header) + sizeof(struct t2audio_msg_base); (void)offset; \
+ if (((struct t2audio_msg_base *) ((struct t2audio_msg_header *) msg->data + 1))->msg != type) \
+ return -EINVAL;
+#define READ_DEVID_VAR(devid) *devid = ((struct t2audio_msg_header *) msg->data)->device_id
+#define READ_VAL(type) ({ offset += sizeof(type); *((type *) ((u8 *) msg->data + offset - sizeof(type))); })
+#define READ_VAR(type, var) *var = READ_VAL(type)
+
+int t2audio_msg_read_start_io_response(struct t2audio_msg *msg)
+{
+ READ_START(T2AUDIO_MSG_START_IO_RESPONSE);
+ return 0;
+}
+
+int t2audio_msg_read_stop_io_response(struct t2audio_msg *msg)
+{
+ READ_START(T2AUDIO_MSG_STOP_IO_RESPONSE);
+ return 0;
+}
+
+int t2audio_msg_read_update_timestamp(struct t2audio_msg *msg, t2audio_device_id_t *devid,
+ u64 *timestamp, u64 *update_seed)
+{
+ READ_START(T2AUDIO_MSG_UPDATE_TIMESTAMP);
+ READ_DEVID_VAR(devid);
+ READ_VAR(u64, timestamp);
+ READ_VAR(u64, update_seed);
+ return 0;
+}
+
+int t2audio_msg_read_get_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj,
+ struct t2audio_prop_addr *prop, void **data, u64 *data_size)
+{
+ READ_START(T2AUDIO_MSG_GET_PROPERTY_RESPONSE);
+ READ_VAR(t2audio_object_id_t, obj);
+ READ_VAR(u32, &prop->element);
+ READ_VAR(u32, &prop->scope);
+ READ_VAR(u32, &prop->selector);
+ READ_VAR(u64, data_size);
+ *data = ((u8 *) msg->data + offset);
+ /* offset += data_size; */
+ return 0;
+}
+
+int t2audio_msg_read_set_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj)
+{
+ READ_START(T2AUDIO_MSG_SET_PROPERTY_RESPONSE);
+ READ_VAR(t2audio_object_id_t, obj);
+ return 0;
+}
+
+int t2audio_msg_read_property_listener_response(struct t2audio_msg *msg, t2audio_object_id_t *obj,
+ struct t2audio_prop_addr *prop)
+{
+ READ_START(T2AUDIO_MSG_PROPERTY_LISTENER_RESPONSE);
+ READ_VAR(t2audio_object_id_t, obj);
+ READ_VAR(u32, &prop->element);
+ READ_VAR(u32, &prop->scope);
+ READ_VAR(u32, &prop->selector);
+ return 0;
+}
+
+int t2audio_msg_read_property_changed(struct t2audio_msg *msg, t2audio_device_id_t *devid, t2audio_object_id_t *obj,
+ struct t2audio_prop_addr *prop)
+{
+ READ_START(T2AUDIO_MSG_PROPERTY_CHANGED);
+ READ_DEVID_VAR(devid);
+ READ_VAR(t2audio_object_id_t, obj);
+ READ_VAR(u32, &prop->element);
+ READ_VAR(u32, &prop->scope);
+ READ_VAR(u32, &prop->selector);
+ return 0;
+}
+
+int t2audio_msg_read_set_input_stream_address_ranges_response(struct t2audio_msg *msg)
+{
+ READ_START(T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE);
+ return 0;
+}
+
+int t2audio_msg_read_get_input_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt)
+{
+ READ_START(T2AUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE);
+ READ_VAR(u64, str_cnt);
+ *str_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset);
+ /* offset += str_cnt * sizeof(t2audio_object_id_t); */
+ return 0;
+}
+
+int t2audio_msg_read_get_output_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt)
+{
+ READ_START(T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE);
+ READ_VAR(u64, str_cnt);
+ *str_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset);
+ /* offset += str_cnt * sizeof(t2audio_object_id_t); */
+ return 0;
+}
+
+int t2audio_msg_read_set_remote_access_response(struct t2audio_msg *msg)
+{
+ READ_START(T2AUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE);
+ return 0;
+}
+
+int t2audio_msg_read_get_device_list_response(struct t2audio_msg *msg, t2audio_device_id_t **dev_l, u64 *dev_cnt)
+{
+ READ_START(T2AUDIO_MSG_GET_DEVICE_LIST_RESPONSE);
+ READ_VAR(u64, dev_cnt);
+ *dev_l = (t2audio_device_id_t *) ((u8 *) msg->data + offset);
+ /* offset += dev_cnt * sizeof(t2audio_device_id_t); */
+ return 0;
+}
+
+#define WRITE_START_OF_TYPE(typev, devid) \
+ size_t offset = sizeof(struct t2audio_msg_header); (void) offset; \
+ ((struct t2audio_msg_header *) msg->data)->type = (typev); \
+ ((struct t2audio_msg_header *) msg->data)->device_id = (devid);
+#define WRITE_START_COMMAND(devid) WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_COMMAND, devid)
+#define WRITE_START_RESPONSE() WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_RESPONSE, 0)
+#define WRITE_START_NOTIFICATION() WRITE_START_OF_TYPE(T2AUDIO_MSG_TYPE_NOTIFICATION, 0)
+#define WRITE_VAL(type, value) { *((type *) ((u8 *) msg->data + offset)) = value; offset += sizeof(value); }
+#define WRITE_BIN(value, size) { memcpy((u8 *) msg->data + offset, value, size); offset += size; }
+#define WRITE_BASE(type) WRITE_VAL(u32, type) WRITE_VAL(u32, 0)
+#define WRITE_END() { msg->size = offset; }
+
+void t2audio_msg_write_start_io(struct t2audio_msg *msg, t2audio_device_id_t dev)
+{
+ WRITE_START_COMMAND(dev);
+ WRITE_BASE(T2AUDIO_MSG_START_IO);
+ WRITE_END();
+}
+
+void t2audio_msg_write_stop_io(struct t2audio_msg *msg, t2audio_device_id_t dev)
+{
+ WRITE_START_COMMAND(dev);
+ WRITE_BASE(T2AUDIO_MSG_STOP_IO);
+ WRITE_END();
+}
+
+void t2audio_msg_write_get_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size)
+{
+ WRITE_START_COMMAND(dev);
+ WRITE_BASE(T2AUDIO_MSG_GET_PROPERTY);
+ WRITE_VAL(t2audio_object_id_t, obj);
+ WRITE_VAL(u32, prop.element);
+ WRITE_VAL(u32, prop.scope);
+ WRITE_VAL(u32, prop.selector);
+ WRITE_VAL(u64, qualifier_size);
+ WRITE_BIN(qualifier, qualifier_size);
+ WRITE_END();
+}
+
+void t2audio_msg_write_set_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size)
+{
+ WRITE_START_COMMAND(dev);
+ WRITE_BASE(T2AUDIO_MSG_SET_PROPERTY);
+ WRITE_VAL(t2audio_object_id_t, obj);
+ WRITE_VAL(u32, prop.element);
+ WRITE_VAL(u32, prop.scope);
+ WRITE_VAL(u32, prop.selector);
+ WRITE_VAL(u64, data_size);
+ WRITE_BIN(data, data_size);
+ WRITE_VAL(u64, qualifier_size);
+ WRITE_BIN(qualifier, qualifier_size);
+ WRITE_END();
+}
+
+void t2audio_msg_write_property_listener(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop)
+{
+ WRITE_START_COMMAND(dev);
+ WRITE_BASE(T2AUDIO_MSG_PROPERTY_LISTENER);
+ WRITE_VAL(t2audio_object_id_t, obj);
+ WRITE_VAL(u32, prop.element);
+ WRITE_VAL(u32, prop.scope);
+ WRITE_VAL(u32, prop.selector);
+ WRITE_END();
+}
+
+void t2audio_msg_write_set_input_stream_address_ranges(struct t2audio_msg *msg, t2audio_device_id_t devid)
+{
+ WRITE_START_COMMAND(devid);
+ WRITE_BASE(T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES);
+ WRITE_END();
+}
+
+void t2audio_msg_write_get_input_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid)
+{
+ WRITE_START_COMMAND(devid);
+ WRITE_BASE(T2AUDIO_MSG_GET_INPUT_STREAM_LIST);
+ WRITE_END();
+}
+
+void t2audio_msg_write_get_output_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid)
+{
+ WRITE_START_COMMAND(devid);
+ WRITE_BASE(T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST);
+ WRITE_END();
+}
+
+void t2audio_msg_write_set_remote_access(struct t2audio_msg *msg, u64 mode)
+{
+ WRITE_START_COMMAND(0);
+ WRITE_BASE(T2AUDIO_MSG_SET_REMOTE_ACCESS);
+ WRITE_VAL(u64, mode);
+ WRITE_END();
+}
+
+void t2audio_msg_write_alive_notification(struct t2audio_msg *msg, u32 proto_ver, u32 msg_ver)
+{
+ WRITE_START_NOTIFICATION();
+ WRITE_BASE(T2AUDIO_MSG_NOTIFICATION_ALIVE);
+ WRITE_VAL(u32, proto_ver);
+ WRITE_VAL(u32, msg_ver);
+ WRITE_END();
+}
+
+void t2audio_msg_write_update_timestamp_response(struct t2audio_msg *msg)
+{
+ WRITE_START_RESPONSE();
+ WRITE_BASE(T2AUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE);
+ WRITE_END();
+}
+
+void t2audio_msg_write_get_device_list(struct t2audio_msg *msg)
+{
+ WRITE_START_COMMAND(0);
+ WRITE_BASE(T2AUDIO_MSG_GET_DEVICE_LIST);
+ WRITE_END();
+}
+
+#define CMD_SHARED_VARS_NO_REPLY \
+ int status = 0; \
+ struct t2audio_send_ctx sctx;
+#define CMD_SHARED_VARS \
+ CMD_SHARED_VARS_NO_REPLY \
+ struct t2audio_msg reply = t2audio_reply_alloc(); \
+ struct t2audio_msg *buf = &reply;
+#define CMD_SEND_REQUEST(fn, ...) \
+ if ((status = t2audio_send_cmd_sync(a, &sctx, buf, 500, fn, ##__VA_ARGS__))) \
+ return status;
+#define CMD_DEF_SHARED_AND_SEND(fn, ...) \
+ CMD_SHARED_VARS \
+ CMD_SEND_REQUEST(fn, ##__VA_ARGS__);
+#define CMD_DEF_SHARED_NO_REPLY_AND_SEND(fn, ...) \
+ CMD_SHARED_VARS_NO_REPLY \
+ CMD_SEND_REQUEST(fn, ##__VA_ARGS__);
+#define CMD_HNDL_REPLY_NO_FREE(fn, ...) \
+ status = fn(buf, ##__VA_ARGS__); \
+ return status;
+#define CMD_HNDL_REPLY_AND_FREE(fn, ...) \
+ status = fn(buf, ##__VA_ARGS__); \
+ t2audio_reply_free(&reply); \
+ return status;
+
+int t2audio_cmd_start_io(struct t2audio_device *a, t2audio_device_id_t devid)
+{
+ CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_start_io, devid);
+ CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_start_io_response);
+}
+int t2audio_cmd_stop_io(struct t2audio_device *a, t2audio_device_id_t devid)
+{
+ CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_stop_io, devid);
+ CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_stop_io_response);
+}
+int t2audio_cmd_get_property(struct t2audio_device *a, struct t2audio_msg *buf,
+ t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size)
+{
+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_property, devid, obj, prop, qualifier, qualifier_size);
+ CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_property_response, &obj, &prop, data, data_size);
+}
+int t2audio_cmd_get_primitive_property(struct t2audio_device *a,
+ t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size)
+{
+ int status;
+ struct t2audio_msg reply = t2audio_reply_alloc();
+ void *r_data;
+ u64 r_data_size;
+ if ((status = t2audio_cmd_get_property(a, &reply, devid, obj, prop, qualifier, qualifier_size,
+ &r_data, &r_data_size)))
+ goto finish;
+ if (r_data_size != data_size) {
+ status = -EINVAL;
+ goto finish;
+ }
+ memcpy(data, r_data, data_size);
+finish:
+ t2audio_reply_free(&reply);
+ return status;
+}
+int t2audio_cmd_set_property(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size)
+{
+ CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_property, devid, obj, prop, data, data_size,
+ qualifier, qualifier_size);
+ CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_property_response, &obj);
+}
+int t2audio_cmd_property_listener(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop)
+{
+ CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_property_listener, devid, obj, prop);
+ CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_property_listener_response, &obj, &prop);
+}
+int t2audio_cmd_set_input_stream_address_ranges(struct t2audio_device *a, t2audio_device_id_t devid)
+{
+ CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_input_stream_address_ranges, devid);
+ CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_input_stream_address_ranges_response);
+}
+int t2audio_cmd_get_input_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid,
+ t2audio_object_id_t **str_l, u64 *str_cnt)
+{
+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_input_stream_list, devid);
+ CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_input_stream_list_response, str_l, str_cnt);
+}
+int t2audio_cmd_get_output_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid,
+ t2audio_object_id_t **str_l, u64 *str_cnt)
+{
+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_output_stream_list, devid);
+ CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_output_stream_list_response, str_l, str_cnt);
+}
+int t2audio_cmd_set_remote_access(struct t2audio_device *a, u64 mode)
+{
+ CMD_DEF_SHARED_AND_SEND(t2audio_msg_write_set_remote_access, mode);
+ CMD_HNDL_REPLY_AND_FREE(t2audio_msg_read_set_remote_access_response);
+}
+int t2audio_cmd_get_device_list(struct t2audio_device *a, struct t2audio_msg *buf,
+ t2audio_device_id_t **dev_l, u64 *dev_cnt)
+{
+ CMD_DEF_SHARED_NO_REPLY_AND_SEND(t2audio_msg_write_get_device_list);
+ CMD_HNDL_REPLY_NO_FREE(t2audio_msg_read_get_device_list_response, dev_l, dev_cnt);
+}
\ No newline at end of file
diff --git a/drivers/staging/t2bce/t2bce_audio/protocol.h b/drivers/staging/t2bce/t2bce_audio/protocol.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/protocol.h
@@ -0,0 +1,147 @@
+#ifndef T2AUDIO_PROTOCOL_H
+#define T2AUDIO_PROTOCOL_H
+
+#include <linux/types.h>
+
+struct t2audio_device;
+
+typedef u64 t2audio_device_id_t;
+typedef u64 t2audio_object_id_t;
+
+struct t2audio_msg {
+ void *data;
+ size_t size;
+};
+
+struct __attribute__((packed)) t2audio_msg_header {
+ char tag[4];
+ u8 type;
+ t2audio_device_id_t device_id;
+};
+struct __attribute__((packed)) t2audio_msg_base {
+ u32 msg;
+ u32 status;
+};
+
+struct t2audio_prop_addr {
+ u32 scope;
+ u32 selector;
+ u32 element;
+};
+#define T2AUDIO_PROP(scope, sel, el) (struct t2audio_prop_addr) { scope, sel, el }
+
+enum {
+ T2AUDIO_MSG_TYPE_COMMAND = 1,
+ T2AUDIO_MSG_TYPE_RESPONSE = 2,
+ T2AUDIO_MSG_TYPE_NOTIFICATION = 3
+};
+
+enum {
+ T2AUDIO_MSG_START_IO = 0,
+ T2AUDIO_MSG_START_IO_RESPONSE = 1,
+ T2AUDIO_MSG_STOP_IO = 2,
+ T2AUDIO_MSG_STOP_IO_RESPONSE = 3,
+ T2AUDIO_MSG_UPDATE_TIMESTAMP = 4,
+ T2AUDIO_MSG_GET_PROPERTY = 7,
+ T2AUDIO_MSG_GET_PROPERTY_RESPONSE = 8,
+ T2AUDIO_MSG_SET_PROPERTY = 9,
+ T2AUDIO_MSG_SET_PROPERTY_RESPONSE = 10,
+ T2AUDIO_MSG_PROPERTY_LISTENER = 11,
+ T2AUDIO_MSG_PROPERTY_LISTENER_RESPONSE = 12,
+ T2AUDIO_MSG_PROPERTY_CHANGED = 13,
+ T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES = 18,
+ T2AUDIO_MSG_SET_INPUT_STREAM_ADDRESS_RANGES_RESPONSE = 19,
+ T2AUDIO_MSG_GET_INPUT_STREAM_LIST = 24,
+ T2AUDIO_MSG_GET_INPUT_STREAM_LIST_RESPONSE = 25,
+ T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST = 26,
+ T2AUDIO_MSG_GET_OUTPUT_STREAM_LIST_RESPONSE = 27,
+ T2AUDIO_MSG_SET_REMOTE_ACCESS = 32,
+ T2AUDIO_MSG_SET_REMOTE_ACCESS_RESPONSE = 33,
+ T2AUDIO_MSG_UPDATE_TIMESTAMP_RESPONSE = 34,
+
+ T2AUDIO_MSG_NOTIFICATION_ALIVE = 100,
+ T2AUDIO_MSG_GET_DEVICE_LIST = 101,
+ T2AUDIO_MSG_GET_DEVICE_LIST_RESPONSE = 102,
+ T2AUDIO_MSG_NOTIFICATION_BOOT = 104
+};
+
+enum {
+ T2AUDIO_REMOTE_ACCESS_OFF = 0,
+ T2AUDIO_REMOTE_ACCESS_ON = 2
+};
+
+enum {
+ T2AUDIO_PROP_SCOPE_GLOBAL = 0x676c6f62, // 'glob'
+ T2AUDIO_PROP_SCOPE_INPUT = 0x696e7074, // 'inpt'
+ T2AUDIO_PROP_SCOPE_OUTPUT = 0x6f757470 // 'outp'
+};
+
+enum {
+ T2AUDIO_PROP_UID = 0x75696420, // 'uid '
+ T2AUDIO_PROP_BOOL_VALUE = 0x6263766c, // 'bcvl'
+ T2AUDIO_PROP_JACK_PLUGGED = 0x6a61636b, // 'jack'
+ T2AUDIO_PROP_SEL_VOLUME = 0x64656176, // 'deav'
+ T2AUDIO_PROP_LATENCY = 0x6c746e63, // 'ltnc'
+ T2AUDIO_PROP_PHYS_FORMAT = 0x70667420 // 'pft '
+};
+
+int t2audio_msg_read_base(struct t2audio_msg *msg, struct t2audio_msg_base *base);
+
+int t2audio_msg_read_start_io_response(struct t2audio_msg *msg);
+int t2audio_msg_read_stop_io_response(struct t2audio_msg *msg);
+int t2audio_msg_read_update_timestamp(struct t2audio_msg *msg, t2audio_device_id_t *devid,
+ u64 *timestamp, u64 *update_seed);
+int t2audio_msg_read_get_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj,
+ struct t2audio_prop_addr *prop, void **data, u64 *data_size);
+int t2audio_msg_read_set_property_response(struct t2audio_msg *msg, t2audio_object_id_t *obj);
+int t2audio_msg_read_property_listener_response(struct t2audio_msg *msg,t2audio_object_id_t *obj,
+ struct t2audio_prop_addr *prop);
+int t2audio_msg_read_property_changed(struct t2audio_msg *msg, t2audio_device_id_t *devid, t2audio_object_id_t *obj,
+ struct t2audio_prop_addr *prop);
+int t2audio_msg_read_set_input_stream_address_ranges_response(struct t2audio_msg *msg);
+int t2audio_msg_read_get_input_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt);
+int t2audio_msg_read_get_output_stream_list_response(struct t2audio_msg *msg, t2audio_object_id_t **str_l, u64 *str_cnt);
+int t2audio_msg_read_set_remote_access_response(struct t2audio_msg *msg);
+int t2audio_msg_read_get_device_list_response(struct t2audio_msg *msg, t2audio_device_id_t **dev_l, u64 *dev_cnt);
+
+void t2audio_msg_write_start_io(struct t2audio_msg *msg, t2audio_device_id_t dev);
+void t2audio_msg_write_stop_io(struct t2audio_msg *msg, t2audio_device_id_t dev);
+void t2audio_msg_write_get_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size);
+void t2audio_msg_write_set_property(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *data, u64 data_size, void *qualifier, u64 qualifier_size);
+void t2audio_msg_write_property_listener(struct t2audio_msg *msg, t2audio_device_id_t dev, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop);
+void t2audio_msg_write_set_input_stream_address_ranges(struct t2audio_msg *msg, t2audio_device_id_t devid);
+void t2audio_msg_write_get_input_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid);
+void t2audio_msg_write_get_output_stream_list(struct t2audio_msg *msg, t2audio_device_id_t devid);
+void t2audio_msg_write_set_remote_access(struct t2audio_msg *msg, u64 mode);
+void t2audio_msg_write_alive_notification(struct t2audio_msg *msg, u32 proto_ver, u32 msg_ver);
+void t2audio_msg_write_update_timestamp_response(struct t2audio_msg *msg);
+void t2audio_msg_write_get_device_list(struct t2audio_msg *msg);
+
+
+int t2audio_cmd_start_io(struct t2audio_device *a, t2audio_device_id_t devid);
+int t2audio_cmd_stop_io(struct t2audio_device *a, t2audio_device_id_t devid);
+int t2audio_cmd_get_property(struct t2audio_device *a, struct t2audio_msg *buf,
+ t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void **data, u64 *data_size);
+int t2audio_cmd_get_primitive_property(struct t2audio_device *a,
+ t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size);
+int t2audio_cmd_set_property(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop, void *qualifier, u64 qualifier_size, void *data, u64 data_size);
+int t2audio_cmd_property_listener(struct t2audio_device *a, t2audio_device_id_t devid, t2audio_object_id_t obj,
+ struct t2audio_prop_addr prop);
+int t2audio_cmd_set_input_stream_address_ranges(struct t2audio_device *a, t2audio_device_id_t devid);
+int t2audio_cmd_get_input_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid,
+ t2audio_object_id_t **str_l, u64 *str_cnt);
+int t2audio_cmd_get_output_stream_list(struct t2audio_device *a, struct t2audio_msg *buf, t2audio_device_id_t devid,
+ t2audio_object_id_t **str_l, u64 *str_cnt);
+int t2audio_cmd_set_remote_access(struct t2audio_device *a, u64 mode);
+int t2audio_cmd_get_device_list(struct t2audio_device *a, struct t2audio_msg *buf,
+ t2audio_device_id_t **dev_l, u64 *dev_cnt);
+
+
+
+#endif //T2AUDIO_PROTOCOL_H
diff --git a/drivers/staging/t2bce/t2bce_audio/protocol_bce.c b/drivers/staging/t2bce/t2bce_audio/protocol_bce.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/protocol_bce.c
@@ -0,0 +1,260 @@
+#include "protocol_bce.h"
+
+#include "audio.h"
+#include <linux/dma-mapping.h>
+
+static void t2audio_bce_out_queue_completion(struct t2bce_core_queue_sq *sq);
+static void t2audio_bce_in_queue_completion(struct t2bce_core_queue_sq *sq);
+static int t2audio_bce_queue_init(struct t2audio_device *dev, struct t2audio_bce_queue *q, const char *name,
+ enum dma_data_direction direction, t2bce_core_sq_completion cfn);
+void t2audio_bce_in_queue_submit_pending(struct t2audio_bce_queue *q, size_t count);
+static void t2audio_deferred_msg_work(struct work_struct *ws);
+
+int t2audio_bce_init(struct t2audio_device *dev)
+{
+ int status;
+ struct t2audio_bce *bce = &dev->bcem;
+ bce->cq = t2bce_core_create_cq(dev->bce, 0x80);
+ spin_lock_init(&bce->spinlock);
+ if (!bce->cq)
+ return -EINVAL;
+ if ((status = t2audio_bce_queue_init(dev, &bce->qout, "com.apple.BridgeAudio.IntelToARM", DMA_TO_DEVICE,
+ t2audio_bce_out_queue_completion))) {
+ return status;
+ }
+ if ((status = t2audio_bce_queue_init(dev, &bce->qin, "com.apple.BridgeAudio.ARMToIntel", DMA_FROM_DEVICE,
+ t2audio_bce_in_queue_completion))) {
+ return status;
+ }
+ t2audio_bce_in_queue_submit_pending(&bce->qin, bce->qin.el_count);
+ return 0;
+}
+
+int t2audio_bce_queue_init(struct t2audio_device *dev, struct t2audio_bce_queue *q, const char *name,
+ enum dma_data_direction direction, t2bce_core_sq_completion cfn)
+{
+ q->cq = dev->bcem.cq;
+ q->el_size = T2AUDIO_BCE_QUEUE_ELEMENT_SIZE;
+ q->el_count = T2AUDIO_BCE_QUEUE_ELEMENT_COUNT;
+ /* Allocate one extra SQ slot because the BCE ring always keeps one entry empty. */
+ q->sq = t2bce_core_create_sq(dev->bce, q->cq, name, (u32) (q->el_count + 1), direction, cfn, dev);
+ if (!q->sq)
+ return -EINVAL;
+
+ q->data = dma_alloc_coherent(t2bce_core_client_dma_dev(dev->bce), q->el_size * q->el_count, &q->dma_addr, GFP_KERNEL);
+ if (!q->data) {
+ t2bce_core_destroy_sq(dev->bce, q->sq);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static void t2audio_send_create_tag(struct t2audio_bce *b, int *tagn, char tag[4])
+{
+ b->tag_num = (b->tag_num + 1) % T2AUDIO_BCE_QUEUE_TAG_COUNT;
+ *tagn = b->tag_num;
+ tag[0] = 'S';
+ tag[1] = '0' + b->tag_num / 100;
+ tag[2] = '0' + (b->tag_num / 10) % 10;
+ tag[3] = '0' + b->tag_num % 10;
+}
+
+int __t2audio_send_prepare(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, char *tag)
+{
+ int status;
+ size_t index;
+ void *dptr;
+ struct t2audio_msg_header *header;
+ if ((status = t2bce_core_reserve_submission(b->qout.sq, &ctx->timeout)))
+ return status;
+ spin_lock_irqsave(&b->spinlock, ctx->irq_flags);
+ index = b->qout.data_tail;
+ dptr = (u8 *) b->qout.data + index * b->qout.el_size;
+ ctx->msg.data = dptr;
+ header = dptr;
+ if (tag)
+ *((u32 *) header->tag) = *((u32 *) tag);
+ else
+ t2audio_send_create_tag(b, &ctx->tag_n, header->tag);
+ return 0;
+}
+
+void __t2audio_send(struct t2audio_bce *b, struct t2audio_send_ctx *ctx)
+{
+#ifdef DEBUG
+ pr_debug("t2bce_audio: Sending command data\n");
+ print_hex_dump(KERN_DEBUG, "t2bce_audio:OUT ", DUMP_PREFIX_NONE, 32, 1, ctx->msg.data, ctx->msg.size, true);
+#endif
+ t2bce_core_set_next_submission_single(b->qout.sq,
+ b->qout.dma_addr + (dma_addr_t) (ctx->msg.data - b->qout.data), ctx->msg.size);
+ t2bce_core_submit_to_device(b->qout.sq);
+ b->qout.data_tail = (b->qout.data_tail + 1) % b->qout.el_count;
+ spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags);
+}
+
+int __t2audio_send_cmd_sync(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, struct t2audio_msg *reply)
+{
+ struct t2audio_bce_queue_entry ent;
+ DECLARE_COMPLETION_ONSTACK(cmpl);
+ ent.msg = reply;
+ ent.cmpl = &cmpl;
+ b->pending_entries[ctx->tag_n] = &ent;
+ __t2audio_send(b, ctx); /* drops b->spinlock */
+ ctx->timeout = wait_for_completion_timeout(&cmpl, ctx->timeout);
+ if (ctx->timeout == 0) {
+ /*
+ * Timed-out replies may still arrive later; remove the waiter before
+ * returning so a stale reply cannot complete a reused stack object.
+ */
+ spin_lock_irqsave(&b->spinlock, ctx->irq_flags);
+ if (b->pending_entries[ctx->tag_n] == &ent)
+ b->pending_entries[ctx->tag_n] = NULL;
+ spin_unlock_irqrestore(&b->spinlock, ctx->irq_flags);
+ return -ETIMEDOUT;
+ }
+ return 0;
+}
+
+static void t2audio_handle_reply(struct t2audio_bce *b, struct t2audio_msg *reply)
+{
+ const char *tag;
+ int tagn;
+ unsigned long irq_flags;
+ char tag_zero[5];
+ struct t2audio_bce_queue_entry *entry;
+
+ tag = ((struct t2audio_msg_header *) reply->data)->tag;
+ if (tag[0] != 'S') {
+ pr_err("t2bce_audio: unexpected reply tag: %.4s\n", tag);
+ return;
+ }
+ *((u32 *) tag_zero) = *((u32 *) tag);
+ tag_zero[4] = 0;
+ if (kstrtoint(&tag_zero[1], 10, &tagn)) {
+ pr_err("t2bce_audio: reply tag parse failed: %.4s\n", tag);
+ return;
+ }
+
+ spin_lock_irqsave(&b->spinlock, irq_flags);
+ entry = b->pending_entries[tagn];
+ if (entry) {
+ if (reply->size < entry->msg->size)
+ entry->msg->size = reply->size;
+ memcpy(entry->msg->data, reply->data, entry->msg->size);
+ complete(entry->cmpl);
+
+ b->pending_entries[tagn] = NULL;
+ } else {
+ pr_warn_ratelimited("t2bce_audio: stale reply without queued command tag=%.4s\n", tag);
+ }
+ spin_unlock_irqrestore(&b->spinlock, irq_flags);
+}
+
+static void t2audio_bce_out_queue_completion(struct t2bce_core_queue_sq *sq)
+{
+ while (t2bce_core_next_completion(sq)) {
+ t2bce_core_notify_submission_complete(sq);
+ }
+}
+
+static void t2audio_bce_in_queue_handle_msg(struct t2audio_device *a, struct t2audio_msg *msg);
+
+static void t2audio_deferred_msg_work(struct work_struct *ws)
+{
+ struct t2audio_deferred_msg *work = container_of(ws, struct t2audio_deferred_msg, ws);
+ struct t2audio_msg_header *header = work->msg.data;
+
+ if (header->type == T2AUDIO_MSG_TYPE_COMMAND)
+ t2audio_handle_command(work->a, &work->msg);
+ else if (header->type == T2AUDIO_MSG_TYPE_NOTIFICATION)
+ t2audio_handle_notification(work->a, &work->msg);
+
+ kfree(work->msg.data);
+ kfree(work);
+}
+
+static void t2audio_bce_in_queue_completion(struct t2bce_core_queue_sq *sq)
+{
+ struct t2audio_msg msg;
+ struct t2audio_device *dev = t2bce_core_queue_sq_userdata(sq);
+ struct t2audio_bce_queue *q = &dev->bcem.qin;
+ struct t2bce_core_sq_completion_data *c;
+ size_t cnt = 0;
+
+ mb();
+ while ((c = t2bce_core_next_completion(sq))) {
+ msg.data = (u8 *) q->data + q->data_head * q->el_size;
+ msg.size = c->data_size;
+#ifdef DEBUG
+ pr_debug("t2bce_audio: Received command data %llx\n", c->data_size);
+ print_hex_dump(KERN_DEBUG, "t2bce_audio:IN ", DUMP_PREFIX_NONE, 32, 1, msg.data, min(msg.size, 128UL), true);
+#endif
+ t2audio_bce_in_queue_handle_msg(dev, &msg);
+
+ q->data_head = (q->data_head + 1) % q->el_count;
+
+ t2bce_core_notify_submission_complete(sq);
+ ++cnt;
+ }
+ t2audio_bce_in_queue_submit_pending(q, cnt);
+}
+
+static void t2audio_bce_in_queue_handle_msg(struct t2audio_device *a, struct t2audio_msg *msg)
+{
+ struct t2audio_msg_header *header = (struct t2audio_msg_header *) msg->data;
+ struct t2audio_deferred_msg *work;
+
+ if (msg->size < sizeof(struct t2audio_msg_header)) {
+ pr_err("t2bce_audio: Msg size smaller than header (%lx)", msg->size);
+ return;
+ }
+ if (header->type == T2AUDIO_MSG_TYPE_RESPONSE) {
+ t2audio_handle_reply(&a->bcem, msg);
+ return;
+ }
+
+ work = kzalloc(sizeof(*work), GFP_ATOMIC);
+ if (!work) {
+ pr_err("t2bce_audio: Failed to allocate deferred message work\n");
+ return;
+ }
+
+ work->msg.data = kmemdup(msg->data, msg->size, GFP_ATOMIC);
+ if (!work->msg.data) {
+ kfree(work);
+ pr_err("t2bce_audio: Failed to copy deferred message\n");
+ return;
+ }
+
+ work->a = a;
+ work->msg.size = msg->size;
+ INIT_WORK(&work->ws, t2audio_deferred_msg_work);
+ schedule_work(&work->ws);
+}
+
+void t2audio_bce_in_queue_submit_pending(struct t2audio_bce_queue *q, size_t count)
+{
+ while (count--) {
+ if (t2bce_core_reserve_submission(q->sq, NULL)) {
+ pr_err("t2bce_audio: Failed to reserve an event queue submission\n");
+ break;
+ }
+ t2bce_core_set_next_submission_single(q->sq,
+ q->dma_addr + (dma_addr_t) (q->data_tail * q->el_size), q->el_size);
+ q->data_tail = (q->data_tail + 1) % q->el_count;
+ }
+ t2bce_core_submit_to_device(q->sq);
+}
+
+struct t2audio_msg t2audio_reply_alloc(void)
+{
+ struct t2audio_msg ret;
+ ret.size = T2AUDIO_BCE_QUEUE_ELEMENT_SIZE;
+ ret.data = kmalloc(ret.size, GFP_KERNEL);
+ return ret;
+}
+
+void t2audio_reply_free(struct t2audio_msg *reply)
+{
+ kfree(reply->data);
+}
diff --git a/drivers/staging/t2bce/t2bce_audio/protocol_bce.h b/drivers/staging/t2bce/t2bce_audio/protocol_bce.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_audio/protocol_bce.h
@@ -0,0 +1,72 @@
+#ifndef T2AUDIO_PROTOCOL_BCE_H
+#define T2AUDIO_PROTOCOL_BCE_H
+
+#include "protocol.h"
+#include "t2bce_core_transport.h"
+
+#define T2AUDIO_BCE_QUEUE_ELEMENT_SIZE 0x1000
+#define T2AUDIO_BCE_QUEUE_ELEMENT_COUNT 20
+
+#define T2AUDIO_BCE_QUEUE_TAG_COUNT 1000
+
+struct t2audio_device;
+
+struct t2audio_bce_queue_entry {
+ struct t2audio_msg *msg;
+ struct completion *cmpl;
+};
+struct t2audio_bce_queue {
+ struct t2bce_core_queue_cq *cq;
+ struct t2bce_core_queue_sq *sq;
+ void *data;
+ dma_addr_t dma_addr;
+ size_t data_head, data_tail;
+ size_t el_size, el_count;
+};
+struct t2audio_bce {
+ struct t2bce_core_queue_cq *cq;
+ struct t2audio_bce_queue qin;
+ struct t2audio_bce_queue qout;
+ int tag_num;
+ struct t2audio_bce_queue_entry *pending_entries[T2AUDIO_BCE_QUEUE_TAG_COUNT];
+ struct spinlock spinlock;
+};
+
+struct t2audio_send_ctx {
+ int status;
+ int tag_n;
+ unsigned long irq_flags;
+ struct t2audio_msg msg;
+ unsigned long timeout;
+};
+
+int t2audio_bce_init(struct t2audio_device *dev);
+int __t2audio_send_prepare(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, char *tag);
+void __t2audio_send(struct t2audio_bce *b, struct t2audio_send_ctx *ctx);
+int __t2audio_send_cmd_sync(struct t2audio_bce *b, struct t2audio_send_ctx *ctx, struct t2audio_msg *reply);
+
+#define t2audio_send_with_tag(a, ctx, tag, tout, fn, ...) ({ \
+ (ctx)->timeout = msecs_to_jiffies(tout); \
+ (ctx)->status = __t2audio_send_prepare(&(a)->bcem, (ctx), (tag)); \
+ if (!(ctx)->status) { \
+ fn(&(ctx)->msg, ##__VA_ARGS__); \
+ __t2audio_send(&(a)->bcem, (ctx)); \
+ } \
+ (ctx)->status; \
+})
+#define t2audio_send(a, ctx, tout, fn, ...) t2audio_send_with_tag(a, ctx, NULL, tout, fn, ##__VA_ARGS__)
+
+#define t2audio_send_cmd_sync(a, ctx, reply, tout, fn, ...) ({ \
+ (ctx)->timeout = msecs_to_jiffies(tout); \
+ (ctx)->status = __t2audio_send_prepare(&(a)->bcem, (ctx), NULL); \
+ if (!(ctx)->status) { \
+ fn(&(ctx)->msg, ##__VA_ARGS__); \
+ (ctx)->status = __t2audio_send_cmd_sync(&(a)->bcem, (ctx), (reply)); \
+ } \
+ (ctx)->status; \
+})
+
+struct t2audio_msg t2audio_reply_alloc(void);
+void t2audio_reply_free(struct t2audio_msg *reply);
+
+#endif //T2AUDIO_PROTOCOL_BCE_H
diff --git a/drivers/staging/t2bce/t2bce_core/Makefile b/drivers/staging/t2bce/t2bce_core/Makefile
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_T2BCE_CORE) += t2bce_core.o
+t2bce_core-y := t2bce_main.o mailbox.o transport.o
+
+ccflags-y += -DWITHOUT_NVME_PATCH
+ccflags-y += -I$(src)/include
+ccflags-y += -I$(src)/../t2bce_dma/include
diff --git a/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h b/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/include/t2bce_core_transport.h
@@ -0,0 +1,83 @@
+#ifndef T2BCE_CORE_TRANSPORT_H
+#define T2BCE_CORE_TRANSPORT_H
+
+#include <linux/device.h>
+#include <linux/dma-direction.h>
+#include <linux/types.h>
+
+struct t2bce_core_client;
+struct t2bce_core_queue_cq;
+struct t2bce_core_queue_sq;
+
+typedef void (*t2bce_core_sq_completion)(struct t2bce_core_queue_sq *sq);
+typedef void (*t2bce_core_client_resume_callback)(void *userdata);
+
+struct t2bce_core_client_pm_ops {
+ void (*shutdown)(void *userdata);
+ void (*pm_reset)(void *userdata);
+ void (*pm_prepare)(void *userdata);
+ void (*pm_prepare_no_state)(void *userdata);
+ void (*pm_mark_no_state_resume)(void *userdata);
+ bool (*pm_is_no_state_resume)(void *userdata);
+ void (*pm_complete)(void *userdata);
+};
+
+struct t2bce_core_sq_completion_data {
+ u32 status;
+ u64 data_size;
+ u64 result;
+};
+
+struct t2bce_core_dma_segment {
+ u64 addr;
+ u64 length;
+};
+
+enum t2bce_completion_status {
+ T2BCE_COMPLETION_SUCCESS = 0,
+ T2BCE_COMPLETION_ERROR = 1,
+ T2BCE_COMPLETION_ABORTED = 2,
+ T2BCE_COMPLETION_NO_SPACE = 3,
+ T2BCE_COMPLETION_OVERRUN = 4,
+};
+
+struct t2bce_core_client *t2bce_core_client_get(struct device *dev);
+void t2bce_core_client_put(struct t2bce_core_client *client);
+
+struct device *t2bce_core_device_get(void);
+void t2bce_core_device_put(struct device *dev);
+struct device *t2bce_core_client_dma_dev(struct t2bce_core_client *client);
+bool t2bce_core_client_no_state_resume(struct t2bce_core_client *client);
+
+void t2bce_core_client_set_resume_complete_callback(struct t2bce_core_client *client,
+ t2bce_core_client_resume_callback callback, void *userdata);
+void t2bce_core_client_set_pm_ops(struct t2bce_core_client *client,
+ const struct t2bce_core_client_pm_ops *ops, void *userdata);
+
+struct t2bce_core_queue_cq *t2bce_core_create_cq(struct t2bce_core_client *client, u32 el_count);
+void t2bce_core_destroy_cq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq);
+
+struct t2bce_core_queue_sq *t2bce_core_create_sq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq,
+ const char *name, u32 el_count, enum dma_data_direction direction,
+ t2bce_core_sq_completion compl, void *userdata);
+void t2bce_core_destroy_sq(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq);
+
+void *t2bce_core_queue_sq_userdata(struct t2bce_core_queue_sq *sq);
+
+int t2bce_core_reserve_submission(struct t2bce_core_queue_sq *sq, unsigned long *timeout);
+void t2bce_core_cancel_submission_reservation(struct t2bce_core_queue_sq *sq);
+void t2bce_core_set_next_submission_single(struct t2bce_core_queue_sq *sq, dma_addr_t addr, size_t size);
+void t2bce_core_set_next_submission_segment_list(struct t2bce_core_queue_sq *sq,
+ dma_addr_t segl_addr, size_t segl_size);
+void t2bce_core_submit_to_device(struct t2bce_core_queue_sq *sq);
+void t2bce_core_notify_submission_complete(struct t2bce_core_queue_sq *sq);
+
+struct t2bce_core_sq_completion_data *t2bce_core_next_completion(struct t2bce_core_queue_sq *sq);
+
+u32 t2bce_core_queue_sq_head(struct t2bce_core_queue_sq *sq);
+u32 t2bce_core_queue_sq_tail(struct t2bce_core_queue_sq *sq);
+u32 t2bce_core_queue_sq_available(struct t2bce_core_queue_sq *sq);
+u32 t2bce_core_queue_sq_capacity(struct t2bce_core_queue_sq *sq);
+int t2bce_core_flush_queue(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq);
+
+#endif
diff --git a/drivers/staging/t2bce/t2bce_core/mailbox.c b/drivers/staging/t2bce/t2bce_core/mailbox.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/mailbox.c
@@ -0,0 +1,234 @@
+#include "mailbox.h"
+#include <linux/atomic.h>
+#include "t2bce.h"
+#include <linux/version.h>
+
+#define REG_MBOX_OUT_BASE 0x820
+#define REG_MBOX_REPLY_COUNTER 0x108
+#define REG_MBOX_REPLY_BASE 0x810
+#define REG_TIMESTAMP_BASE 0xC000
+
+#define BCE_MBOX_TIMEOUT_MS 1000
+
+void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb)
+{
+ mb->reg_mb = reg_mb;
+ mutex_init(&mb->channel_lock);
+ init_completion(&mb->mb_completion);
+}
+
+int bce_mailbox_channel_pause(struct bce_mailbox *mb)
+{
+ /*
+ * System PM needs to exclude new mailbox transactions while the core
+ * driver is moving bridgeOS into or out of sleep.
+ */
+ mutex_lock(&mb->channel_lock);
+ return 0;
+}
+
+void bce_mailbox_channel_resume(struct bce_mailbox *mb)
+{
+ mutex_unlock(&mb->channel_lock);
+}
+
+int bce_mailbox_send_locked_timeout(struct bce_mailbox *mb, u64 msg, u64* recv, unsigned int timeout_ms)
+{
+ u32 __iomem *regb;
+ int status = 0;
+
+ if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) {
+ status = -EEXIST;
+ goto out;
+ }
+ reinit_completion(&mb->mb_completion);
+
+ pr_debug("bce_mailbox_send: %llx\n", msg);
+ regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE);
+ iowrite32((u32) msg, regb);
+ iowrite32((u32) (msg >> 32), regb + 1);
+ iowrite32(0, regb + 2);
+ iowrite32(0, regb + 3);
+
+ wait_for_completion_timeout(&mb->mb_completion, msecs_to_jiffies(timeout_ms));
+ if (atomic_read(&mb->mb_status) != 2) {
+ atomic_set(&mb->mb_status, 0);
+ status = -ETIMEDOUT;
+ goto out;
+ }
+
+ *recv = mb->mb_result;
+ pr_debug("bce_mailbox_send: reply %llx\n", *recv);
+
+ atomic_set(&mb->mb_status, 0);
+out:
+ return status;
+}
+
+int bce_mailbox_send_timeout(struct bce_mailbox *mb, u64 msg, u64* recv, unsigned int timeout_ms)
+{
+ int status;
+
+ /* Serialize mailbox traffic; bridgeOS accepts one active command at a time. */
+ mutex_lock(&mb->channel_lock);
+ status = bce_mailbox_send_locked_timeout(mb, msg, recv, timeout_ms);
+ mutex_unlock(&mb->channel_lock);
+ return status;
+}
+
+int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv)
+{
+ return bce_mailbox_send_timeout(mb, msg, recv, BCE_MBOX_TIMEOUT_MS);
+}
+
+int bce_mailbox_send_locked(struct bce_mailbox *mb, u64 msg, u64 *recv)
+{
+ return bce_mailbox_send_locked_timeout(mb, msg, recv, BCE_MBOX_TIMEOUT_MS);
+}
+
+int bce_mailbox_send_no_reply_locked(struct bce_mailbox *mb, u64 msg)
+{
+ u32 __iomem *regb;
+ int status = 0;
+
+ if (atomic_cmpxchg(&mb->mb_status, 0, 1) != 0) {
+ status = -EEXIST;
+ goto out;
+ }
+
+ pr_debug("bce_mailbox_send_no_reply: %llx\n", msg);
+ regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_OUT_BASE);
+ iowrite32((u32) msg, regb);
+ iowrite32((u32) (msg >> 32), regb + 1);
+ iowrite32(0, regb + 2);
+ iowrite32(0, regb + 3);
+
+ atomic_set(&mb->mb_status, 0);
+out:
+ return status;
+}
+
+int bce_mailbox_send_no_reply(struct bce_mailbox *mb, u64 msg)
+{
+ int status;
+
+ /* Fire-and-forget commands share the same mailbox serialization. */
+ mutex_lock(&mb->channel_lock);
+ status = bce_mailbox_send_no_reply_locked(mb, msg);
+ mutex_unlock(&mb->channel_lock);
+ return status;
+}
+
+static int bce_mailbox_retrive_response(struct bce_mailbox *mb)
+{
+ u32 __iomem *regb;
+ u32 lo, hi;
+ int count, counter;
+ u32 res = ioread32((u8*) mb->reg_mb + REG_MBOX_REPLY_COUNTER);
+ count = (res >> 20) & 0xf;
+ counter = count;
+ pr_debug("bce_mailbox_retrive_response count=%i\n", count);
+ while (counter--) {
+ regb = (u32*) ((u8*) mb->reg_mb + REG_MBOX_REPLY_BASE);
+ lo = ioread32(regb);
+ hi = ioread32(regb + 1);
+ ioread32(regb + 2);
+ ioread32(regb + 3);
+ pr_debug("bce_mailbox_retrive_response %llx\n", ((u64) hi << 32) | lo);
+ mb->mb_result = ((u64) hi << 32) | lo;
+ }
+ return count > 0 ? 0 : -ENODATA;
+}
+
+int bce_mailbox_handle_interrupt(struct bce_mailbox *mb)
+{
+ int status = bce_mailbox_retrive_response(mb);
+ if (!status) {
+ if (atomic_read(&mb->mb_status) == 1) {
+ atomic_set(&mb->mb_status, 2);
+ complete(&mb->mb_completion);
+ } else {
+ pr_debug("bce_mailbox_handle_interrupt: unsolicited reply %llx\n", mb->mb_result);
+ atomic_set(&mb->mb_status, 0);
+ }
+ }
+ return status;
+}
+
+static void bce_xhci_pm_tick(struct timer_list *tl);
+
+void bce_xhci_pm_init(struct bce_xhci_pm *pm, void __iomem *reg)
+{
+ u32 __iomem *regb;
+
+ spin_lock_init(&pm->stop_sl);
+ pm->stopped = false;
+
+ pm->reg = reg;
+
+ regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE);
+
+ ioread32(regb);
+ mb();
+
+ timer_setup(&pm->timer, bce_xhci_pm_tick, 0);
+}
+
+void bce_xhci_pm_start(struct bce_xhci_pm *pm, bool is_initial)
+{
+ unsigned long flags;
+ u32 __iomem *regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE);
+
+ if (is_initial) {
+ iowrite32((u32) -4, regb + 2);
+ iowrite32((u32) -1, regb);
+ } else {
+ iowrite32((u32) -3, regb + 2);
+ iowrite32((u32) -1, regb);
+ }
+
+ spin_lock_irqsave(&pm->stop_sl, flags);
+ pm->stopped = false;
+ spin_unlock_irqrestore(&pm->stop_sl, flags);
+ mod_timer(&pm->timer, jiffies + msecs_to_jiffies(150));
+}
+
+void bce_xhci_pm_stop(struct bce_xhci_pm *pm)
+{
+ unsigned long flags;
+ u32 __iomem *regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE);
+
+ spin_lock_irqsave(&pm->stop_sl, flags);
+ pm->stopped = true;
+ spin_unlock_irqrestore(&pm->stop_sl, flags);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,15,0)
+ del_timer_sync(&pm->timer);
+#else
+ timer_delete_sync(&pm->timer);
+#endif
+ iowrite32((u32) -2, regb + 2);
+ iowrite32((u32) -1, regb);
+}
+
+static void bce_xhci_pm_tick(struct timer_list *tl)
+{
+ struct bce_xhci_pm *pm;
+ unsigned long flags;
+ u32 __iomem *regb;
+ ktime_t bt;
+
+ pm = container_of(tl, struct bce_xhci_pm, timer);
+ regb = (u32*) ((u8*) pm->reg + REG_TIMESTAMP_BASE);
+ local_irq_save(flags);
+ ioread32(regb + 2);
+ mb();
+ bt = ktime_get_boottime();
+ iowrite32((u32) bt, regb + 2);
+ iowrite32((u32) (bt >> 32), regb);
+
+ spin_lock(&pm->stop_sl);
+ if (!pm->stopped)
+ mod_timer(&pm->timer, jiffies + msecs_to_jiffies(150));
+ spin_unlock(&pm->stop_sl);
+ local_irq_restore(flags);
+}
diff --git a/drivers/staging/t2bce/t2bce_core/mailbox.h b/drivers/staging/t2bce/t2bce_core/mailbox.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/mailbox.h
@@ -0,0 +1,64 @@
+#ifndef BCE_MAILBOX_H
+#define BCE_MAILBOX_H
+
+#include <linux/completion.h>
+#include <linux/mutex.h>
+#include <linux/pci.h>
+#include <linux/timer.h>
+
+struct bce_mailbox {
+ void __iomem *reg_mb;
+ struct mutex channel_lock;
+
+ atomic_t mb_status; // possible statuses: 0 (no msg), 1 (has active msg), 2 (got reply)
+ struct completion mb_completion;
+ uint64_t mb_result;
+};
+
+enum bce_message_type {
+ BCE_MB_REGISTER_COMMAND_SQ = 0x7, // to-device
+ BCE_MB_REGISTER_COMMAND_CQ = 0x8, // to-device
+ BCE_MB_REGISTER_COMMAND_QUEUE_REPLY = 0xB, // to-host
+ BCE_MB_SET_FW_PROTOCOL_VERSION = 0xC, // both
+ BCE_MB_SLEEP_NO_STATE = 0x14, // to-device
+ BCE_MB_RESTORE_NO_STATE = 0x15, // to-device
+ BCE_MB_SLEEP_LIGHT = 0x16, // to-device, unused; reserved for future runtime-PM work
+ BCE_MB_SAVE_STATE_AND_SLEEP = 0x17, // to-device
+ BCE_MB_RESTORE_STATE_AND_WAKE = 0x18, // to-device
+ BCE_MB_SAVE_STATE_AND_SLEEP_REJECTED = 0x19, // from-device, stateful suspend payload rejected
+ BCE_MB_SAVE_RESTORE_STATE_COMPLETE = 0x1A, // from-device
+};
+
+#define BCE_MB_MSG(type, value) (((u64) (type) << 58) | ((value) & 0x3FFFFFFFFFFFFFFLL))
+#define BCE_MB_TYPE(v) ((u32) (v >> 58))
+#define BCE_MB_VALUE(v) (v & 0x3FFFFFFFFFFFFFFLL)
+
+void bce_mailbox_init(struct bce_mailbox *mb, void __iomem *reg_mb);
+
+int bce_mailbox_channel_pause(struct bce_mailbox *mb);
+void bce_mailbox_channel_resume(struct bce_mailbox *mb);
+
+int bce_mailbox_send(struct bce_mailbox *mb, u64 msg, u64* recv);
+int bce_mailbox_send_timeout(struct bce_mailbox *mb, u64 msg, u64 *recv, unsigned int timeout_ms);
+int bce_mailbox_send_no_reply(struct bce_mailbox *mb, u64 msg);
+int bce_mailbox_send_locked(struct bce_mailbox *mb, u64 msg, u64 *recv);
+int bce_mailbox_send_locked_timeout(struct bce_mailbox *mb, u64 msg, u64 *recv, unsigned int timeout_ms);
+int bce_mailbox_send_no_reply_locked(struct bce_mailbox *mb, u64 msg);
+
+int bce_mailbox_handle_interrupt(struct bce_mailbox *mb);
+
+
+struct bce_xhci_pm {
+ void __iomem *reg;
+ struct timer_list timer;
+ struct spinlock stop_sl;
+ bool stopped;
+};
+
+void bce_xhci_pm_init(struct bce_xhci_pm *pm, void __iomem *reg);
+
+void bce_xhci_pm_start(struct bce_xhci_pm *pm, bool is_initial);
+
+void bce_xhci_pm_stop(struct bce_xhci_pm *pm);
+
+#endif //BCEDRIVER_MAILBOX_H
diff --git a/drivers/staging/t2bce/t2bce_core/t2bce.h b/drivers/staging/t2bce/t2bce_core/t2bce.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/t2bce.h
@@ -0,0 +1,51 @@
+#ifndef APPLE_BCE_H
+#define APPLE_BCE_H
+
+#include <linux/pci.h>
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/list.h>
+#include <linux/srcu.h>
+#include "mailbox.h"
+#include "t2bce_dma_queue.h"
+
+#define BC_PROTOCOL_VERSION 0x20001
+
+struct t2bce_device {
+ struct pci_dev *pci, *pci0;
+ dev_t devt;
+ struct device *dev;
+ void __iomem *reg_mem_mb;
+ void __iomem *reg_mem_dma;
+ struct t2bce_dma_engine dma;
+ struct bce_mailbox mbox;
+ struct bce_xhci_pm xhci_pm;
+ bool is_being_removed;
+
+ dma_addr_t saved_data_dma_addr;
+ void *saved_data_dma_ptr;
+ size_t saved_data_dma_size;
+ u32 fw_version;
+ bool stateful_suspend_valid;
+ bool no_state_fallback;
+ bool no_state_resume;
+ bool mailbox_channel_active;
+ struct mutex pm_lock;
+ struct mutex clients_lock;
+ struct list_head clients;
+ struct srcu_struct clients_srcu;
+ bool clients_srcu_ready;
+};
+
+extern struct t2bce_device *global_bce;
+
+void t2bce_core_notify_resume_complete(struct t2bce_device *bce);
+void t2bce_core_clients_shutdown(struct t2bce_device *bce);
+void t2bce_core_clients_pm_reset(struct t2bce_device *bce);
+void t2bce_core_clients_pm_prepare(struct t2bce_device *bce);
+void t2bce_core_clients_pm_prepare_no_state(struct t2bce_device *bce);
+void t2bce_core_clients_pm_mark_no_state_resume(struct t2bce_device *bce);
+bool t2bce_core_clients_pm_has_no_state_resume(struct t2bce_device *bce);
+void t2bce_core_clients_pm_complete(struct t2bce_device *bce);
+
+#endif //APPLE_BCE_H
diff --git a/drivers/staging/t2bce/t2bce_core/t2bce_main.c b/drivers/staging/t2bce/t2bce_core/t2bce_main.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/t2bce_main.c
@@ -0,0 +1,766 @@
+#include "t2bce.h"
+#include <linux/module.h>
+#include <linux/crc32.h>
+#include <linux/delay.h>
+#include <linux/version.h>
+
+/*
+ * Apple T2 Macs contain a coprocessor serving as a board supervisor.
+ * The T2 is basically a repurposed iPhone CPU running bridgeOS,
+ * a stripped down version of WatchOS. Apple reuses the audio codec
+ * and USB interfaces of this SOC and uses a "Buffer Copy Engine" (BCE)
+ * for communication between the host OS, the T2 and the VHCI USB devices
+ * behind the T2 USB hub like internal keyboard, trackpad, touchbar etc..
+ *
+ * This core driver owns the BCE PCI device, mailbox, DMA queue registration
+ * and system PM ordering. Other modules bind as clients:
+ * - t2bce_audio exposes the bridgeOS audio endpoints to ALSA.
+ * - t2bce_vhci exposes the T2-internal virtual USB hub and devices to usbcore.
+ * - t2bce_dma provides the queue engine used by both.
+ *
+ * ACPI describes the host-side PCI devices, but bridgeOS still controls the
+ * remote endpoints and must be kept in sync during probe, shutdown and system
+ * suspend/resume. The PM code below mirrors the observed firmware ordering:
+ * It quiesces clients, drains the mailbox channel, asks bridgeOS to save or discard
+ * BCE state, then lets clients rebuild or continue according to the selected
+ * resume path.
+ */
+
+static dev_t bce_chrdev;
+static struct class *bce_class;
+static const bool bce_use_stateful_sleep = true;
+
+struct t2bce_device *global_bce;
+
+static int bce_create_command_queues(struct t2bce_device *bce);
+static void bce_free_command_queues(struct t2bce_device *bce);
+static irqreturn_t bce_handle_mb_irq(int irq, void *dev);
+static irqreturn_t bce_handle_dma_irq(int irq, void *dev);
+static int bce_fw_version_handshake(struct t2bce_device *bce);
+static int bce_register_command_queue(struct t2bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq);
+static int t2bce_dma_register_queue(void *userdata, struct bce_queue_memcfg *cfg,
+ const char *name, bool isdirout);
+static int t2bce_dma_unregister_queue(void *userdata, u16 qid);
+static int t2bce_dma_flush_queue(void *userdata, u16 qid);
+static int bce_alloc_state_buffer(struct t2bce_device *bce);
+static void bce_free_state_buffer(struct t2bce_device *bce);
+static int bce_pm_suspend_prepare(struct t2bce_device *bce);
+static void bce_pm_suspend_abort(struct t2bce_device *bce);
+static void bce_pm_resume_finish(struct t2bce_device *bce);
+
+static const struct t2bce_dma_engine_ops t2bce_dma_ops = {
+ .register_queue = t2bce_dma_register_queue,
+ .unregister_queue = t2bce_dma_unregister_queue,
+ .flush_queue = t2bce_dma_flush_queue,
+};
+
+static int bce_alloc_state_buffer(struct t2bce_device *bce)
+{
+ /* bridgeOS stateful sleep stores BCE state in a persistent 0x2000 buffer. */
+ bce->saved_data_dma_size = 0x2000;
+ bce->saved_data_dma_ptr = dma_alloc_coherent(&bce->pci->dev, bce->saved_data_dma_size,
+ &bce->saved_data_dma_addr, GFP_KERNEL);
+ if (!bce->saved_data_dma_ptr) {
+ bce->saved_data_dma_size = 0;
+ bce->saved_data_dma_addr = 0;
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static void bce_free_state_buffer(struct t2bce_device *bce)
+{
+ if (!bce->saved_data_dma_ptr)
+ return;
+
+ dma_free_coherent(&bce->pci->dev, bce->saved_data_dma_size,
+ bce->saved_data_dma_ptr, bce->saved_data_dma_addr);
+ bce->saved_data_dma_ptr = NULL;
+ bce->saved_data_dma_addr = 0;
+ bce->saved_data_dma_size = 0;
+}
+
+static int t2bce_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+ struct t2bce_device *bce = NULL;
+ int status = 0;
+ int nvec;
+
+ pr_debug("t2bce_core: capturing our device\n");
+
+ if (pci_enable_device(dev))
+ return -ENODEV;
+ if (pci_request_regions(dev, "t2bce_core")) {
+ status = -ENODEV;
+ goto fail;
+ }
+ pci_set_master(dev);
+ nvec = pci_alloc_irq_vectors(dev, 1, 8, PCI_IRQ_MSI);
+ if (nvec < 5) {
+ status = -EINVAL;
+ goto fail;
+ }
+
+ bce = kzalloc(sizeof(struct t2bce_device), GFP_KERNEL);
+ if (!bce) {
+ status = -ENOMEM;
+ goto fail;
+ }
+
+ bce->pci = dev;
+ pci_set_drvdata(dev, bce);
+
+ bce->devt = bce_chrdev;
+ bce->dev = device_create(bce_class, &dev->dev, bce->devt, NULL, "t2bce_core");
+ if (IS_ERR_OR_NULL(bce->dev)) {
+ status = PTR_ERR(bce->dev);
+ goto fail;
+ }
+
+ bce->reg_mem_mb = pci_iomap(dev, 4, 0);
+ bce->reg_mem_dma = pci_iomap(dev, 2, 0);
+
+ if (IS_ERR_OR_NULL(bce->reg_mem_mb) || IS_ERR_OR_NULL(bce->reg_mem_dma)) {
+ dev_warn(&dev->dev, "t2bce_core: Failed to pci_iomap required regions\n");
+ goto fail;
+ }
+
+ bce_mailbox_init(&bce->mbox, bce->reg_mem_mb);
+ bce_xhci_pm_init(&bce->xhci_pm, bce->reg_mem_mb);
+ bce->dma.dma_dev = &dev->dev;
+ bce->dma.ops = &t2bce_dma_ops;
+ bce->dma.ops_userdata = bce;
+ bce->dma.reg_mem_dma = bce->reg_mem_dma;
+
+ spin_lock_init(&bce->dma.queues_lock);
+ ida_init(&bce->dma.queue_ida);
+ mutex_init(&bce->pm_lock);
+ mutex_init(&bce->clients_lock);
+ INIT_LIST_HEAD(&bce->clients);
+ status = init_srcu_struct(&bce->clients_srcu);
+ if (status)
+ goto fail;
+ bce->clients_srcu_ready = true;
+ bce->mailbox_channel_active = true;
+
+ if ((status = pci_request_irq(dev, 0, bce_handle_mb_irq, NULL, dev, "bce_mbox")))
+ goto fail;
+ if ((status = pci_request_irq(dev, 4, NULL, bce_handle_dma_irq, dev, "bce_dma")))
+ goto fail_interrupt_0;
+
+ if ((status = dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(37)))) {
+ dev_warn(&dev->dev, "dma: Setting mask failed\n");
+ goto fail_interrupt;
+ }
+
+ if ((status = bce_alloc_state_buffer(bce)))
+ goto fail_interrupt;
+
+ /*
+ * DMA on the BCE function depends on function 0 also being bus master.
+ * Keep a reference so probe and resume can restore that firmware-visible
+ * host state.
+ */
+ bce->pci0 = pci_get_slot(dev->bus, PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
+#ifndef WITHOUT_NVME_PATCH
+ if ((status = pci_enable_device_mem(bce->pci0))) {
+ dev_warn(&dev->dev, "t2bce_core: failed to enable function 0\n");
+ goto fail_dev0;
+ }
+#endif
+ pci_set_master(bce->pci0);
+
+ /* bridgeOS expects the XHCI PM doorbell before the controller becomes active. */
+ bce_xhci_pm_start(&bce->xhci_pm, true);
+
+ if ((status = bce_fw_version_handshake(bce)))
+ goto fail_ts;
+ pr_debug("t2bce_core: handshake done\n");
+
+ if ((status = bce_create_command_queues(bce))) {
+ pr_err("t2bce_core: Creating command queues failed\n");
+ goto fail_ts;
+ }
+
+ global_bce = bce;
+ pr_info("t2bce_core: initialized\n");
+
+ return 0;
+
+fail_ts:
+ bce_free_state_buffer(bce);
+ bce_xhci_pm_stop(&bce->xhci_pm);
+#ifndef WITHOUT_NVME_PATCH
+ pci_disable_device(bce->pci0);
+fail_dev0:
+#endif
+ pci_dev_put(bce->pci0);
+fail_interrupt:
+ pci_free_irq(dev, 4, dev);
+fail_interrupt_0:
+ pci_free_irq(dev, 0, dev);
+fail:
+ if (bce && bce->dev) {
+ device_destroy(bce_class, bce->devt);
+
+ if (!IS_ERR_OR_NULL(bce->reg_mem_mb))
+ pci_iounmap(dev, bce->reg_mem_mb);
+ if (!IS_ERR_OR_NULL(bce->reg_mem_dma))
+ pci_iounmap(dev, bce->reg_mem_dma);
+
+ if (bce->clients_srcu_ready)
+ cleanup_srcu_struct(&bce->clients_srcu);
+
+ kfree(bce);
+ }
+
+ pci_free_irq_vectors(dev);
+ pci_release_regions(dev);
+ pci_disable_device(dev);
+
+ if (!status)
+ status = -EINVAL;
+ return status;
+}
+
+static int bce_create_command_queues(struct t2bce_device *bce)
+{
+ int status;
+ struct bce_queue_memcfg *cfg;
+ struct t2bce_dma_engine *dma = &bce->dma;
+
+ dma->cmd_cq = t2bce_dma_alloc_cq(dma, 0, 0x20);
+ dma->cmd_cmdq = t2bce_dma_alloc_cmdq(dma, 1, 0x20);
+ if (dma->cmd_cq == NULL || dma->cmd_cmdq == NULL) {
+ status = -ENOMEM;
+ goto err;
+ }
+ dma->queues[0] = (struct bce_queue *) dma->cmd_cq;
+ dma->queues[1] = (struct bce_queue *) dma->cmd_cmdq->sq;
+
+ cfg = kzalloc(sizeof(struct bce_queue_memcfg), GFP_KERNEL);
+ if (!cfg) {
+ status = -ENOMEM;
+ goto err;
+ }
+ t2bce_dma_get_cq_memcfg(dma->cmd_cq, cfg);
+ if ((status = bce_register_command_queue(bce, cfg, false)))
+ goto err;
+ t2bce_dma_get_sq_memcfg(dma->cmd_cmdq->sq, dma->cmd_cq, cfg);
+ if ((status = bce_register_command_queue(bce, cfg, true)))
+ goto err;
+ kfree(cfg);
+
+ return 0;
+
+err:
+ if (dma->cmd_cq)
+ t2bce_dma_free_cq(dma, dma->cmd_cq);
+ if (dma->cmd_cmdq)
+ t2bce_dma_free_cmdq(dma, dma->cmd_cmdq);
+ return status;
+}
+
+static void bce_free_command_queues(struct t2bce_device *bce)
+{
+ struct t2bce_dma_engine *dma = &bce->dma;
+
+ t2bce_dma_free_cq(dma, dma->cmd_cq);
+ t2bce_dma_free_cmdq(dma, dma->cmd_cmdq);
+ dma->cmd_cq = NULL;
+ dma->queues[0] = NULL;
+}
+
+static irqreturn_t bce_handle_mb_irq(int irq, void *dev)
+{
+ struct t2bce_device *bce = pci_get_drvdata(dev);
+ bce_mailbox_handle_interrupt(&bce->mbox);
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t bce_handle_dma_irq(int irq, void *dev)
+{
+ int i;
+ size_t ce = 0;
+ struct t2bce_device *bce = pci_get_drvdata(dev);
+ struct t2bce_dma_engine *dma = &bce->dma;
+
+ spin_lock(&dma->queues_lock);
+ for (i = 0; i < BCE_MAX_QUEUE_COUNT; i++)
+ if (dma->queues[i] && dma->queues[i]->type == BCE_QUEUE_CQ)
+ t2bce_dma_handle_cq_completions_locked(dma, (struct bce_queue_cq *) dma->queues[i], &ce);
+ spin_unlock(&dma->queues_lock);
+ t2bce_dma_dispatch_pending_sq_completions(dma, ce);
+ return IRQ_HANDLED;
+}
+
+static int bce_fw_version_handshake(struct t2bce_device *bce)
+{
+ u64 result;
+ u32 fw_version;
+ int status;
+
+ if ((status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VERSION, BC_PROTOCOL_VERSION),
+ &result)))
+ return status;
+ fw_version = (u32) BCE_MB_VALUE(result);
+ if (BCE_MB_TYPE(result) != BCE_MB_SET_FW_PROTOCOL_VERSION ||
+ fw_version > BC_PROTOCOL_VERSION) {
+ pr_err("t2bce_core: FW version handshake failed %x:%llx\n", BCE_MB_TYPE(result), BCE_MB_VALUE(result));
+ return -EINVAL;
+ }
+ bce->fw_version = fw_version;
+ return 0;
+}
+
+static bool bce_stateful_supported(struct t2bce_device *bce)
+{
+ /* bridgeOS exposes stateful sleep through protocol version 0x20001+. */
+ return bce_use_stateful_sleep && bce->fw_version >= BC_PROTOCOL_VERSION;
+}
+
+static int bce_pm_wait_mailbox_idle(struct t2bce_device *bce)
+{
+ int retries = 100;
+
+ while (atomic_read(&bce->mbox.mb_status) != 0) {
+ if (!--retries) {
+ pr_err("t2bce_core: timeout waiting for BCE mailbox channel to quiesce\n");
+ return -ETIMEDOUT;
+ }
+ usleep_range(5000, 10000);
+ }
+
+ return 0;
+}
+
+static int bce_pm_channel_pause(struct t2bce_device *bce)
+{
+ int status;
+
+ status = bce_mailbox_channel_pause(&bce->mbox);
+ if (status)
+ return status;
+
+ status = bce_pm_wait_mailbox_idle(bce);
+
+ if (status)
+ bce_mailbox_channel_resume(&bce->mbox);
+ if (status)
+ return status;
+
+ /* Keep new mailbox commands out while bridgeOS is entering sleep. */
+ bce->mailbox_channel_active = false;
+ return 0;
+}
+
+static void bce_pm_channel_resume(struct t2bce_device *bce)
+{
+ /* Reopen mailbox traffic after bridgeOS completed the wake handshake. */
+ bce->mailbox_channel_active = true;
+ bce_mailbox_channel_resume(&bce->mbox);
+}
+
+static int bce_pm_suspend_prepare(struct t2bce_device *bce)
+{
+ int status;
+
+ status = bce_pm_channel_pause(bce);
+ if (status)
+ return status;
+
+ /* The XHCI PM doorbell is part of the bridgeOS sleep ordering. */
+ bce_xhci_pm_stop(&bce->xhci_pm);
+ return 0;
+}
+
+static void bce_pm_suspend_abort(struct t2bce_device *bce)
+{
+ /* Failed suspend must unwind the wrapper order back to the running state. */
+ bce_xhci_pm_start(&bce->xhci_pm, false);
+ bce_pm_channel_resume(bce);
+}
+
+static void bce_pm_resume_finish(struct t2bce_device *bce)
+{
+ /* Match the bridgeOS wake ordering before clients observe resume complete. */
+ bce_xhci_pm_start(&bce->xhci_pm, false);
+ bce_pm_channel_resume(bce);
+}
+
+static int bce_register_command_queue(struct t2bce_device *bce, struct bce_queue_memcfg *cfg, int is_sq)
+{
+ int status;
+ int cmd_type;
+ u64 result;
+ /* The firmware accepts a TO_DEVICE mapping for this command payload. */
+ dma_addr_t a = dma_map_single(&bce->pci->dev, cfg, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE);
+ if (dma_mapping_error(&bce->pci->dev, a))
+ return -ENOMEM;
+ cmd_type = is_sq ? BCE_MB_REGISTER_COMMAND_SQ : BCE_MB_REGISTER_COMMAND_CQ;
+ status = bce_mailbox_send(&bce->mbox, BCE_MB_MSG(cmd_type, a), &result);
+ dma_unmap_single(&bce->pci->dev, a, sizeof(struct bce_queue_memcfg), DMA_TO_DEVICE);
+ if (status)
+ return status;
+ if (BCE_MB_TYPE(result) != BCE_MB_REGISTER_COMMAND_QUEUE_REPLY)
+ return -EINVAL;
+ return 0;
+}
+
+static int t2bce_dma_register_queue(void *userdata, struct bce_queue_memcfg *cfg,
+ const char *name, bool isdirout)
+{
+ struct t2bce_device *bce = userdata;
+
+ return t2bce_dma_cmd_register_queue(bce->dma.cmd_cmdq, cfg, name, isdirout);
+}
+
+static int t2bce_dma_unregister_queue(void *userdata, u16 qid)
+{
+ struct t2bce_device *bce = userdata;
+
+ return t2bce_dma_cmd_unregister_memory_queue(bce->dma.cmd_cmdq, qid);
+}
+
+static int t2bce_dma_flush_queue(void *userdata, u16 qid)
+{
+ struct t2bce_device *bce = userdata;
+
+ return t2bce_dma_cmd_flush_memory_queue(bce->dma.cmd_cmdq, qid);
+}
+
+static void t2bce_remove(struct pci_dev *dev)
+{
+ struct t2bce_device *bce = pci_get_drvdata(dev);
+ bce->is_being_removed = true;
+ bce->dma.is_being_removed = true;
+ global_bce = NULL;
+
+ bce_free_state_buffer(bce);
+ bce_xhci_pm_stop(&bce->xhci_pm);
+#ifndef WITHOUT_NVME_PATCH
+ pci_disable_device(bce->pci0);
+#endif
+ pci_dev_put(bce->pci0);
+ pci_free_irq(dev, 0, dev);
+ pci_free_irq(dev, 4, dev);
+ bce_free_command_queues(bce);
+ pci_iounmap(dev, bce->reg_mem_mb);
+ pci_iounmap(dev, bce->reg_mem_dma);
+ device_destroy(bce_class, bce->devt);
+ pci_free_irq_vectors(dev);
+ pci_release_regions(dev);
+ pci_disable_device(dev);
+ cleanup_srcu_struct(&bce->clients_srcu);
+ kfree(bce);
+}
+
+static void t2bce_shutdown(struct pci_dev *dev)
+{
+ struct t2bce_device *bce = pci_get_drvdata(dev);
+ int status = 0;
+
+ if (!bce)
+ return;
+
+ mutex_lock(&bce->pm_lock);
+ bce->is_being_removed = true;
+ bce->dma.is_being_removed = true;
+ bce->stateful_suspend_valid = false;
+ bce->no_state_fallback = false;
+ bce->no_state_resume = false;
+ t2bce_core_clients_pm_reset(bce);
+
+ /*
+ * Do not tear down allocations here;
+ * just leave the T2 side quiet while command queues and mailbox access are
+ * still valid: USB HCD off, XHCI PM sentinel written, mailbox drained.
+ */
+ t2bce_core_clients_shutdown(bce);
+ bce_xhci_pm_stop(&bce->xhci_pm);
+
+ if (bce->mailbox_channel_active) {
+ status = bce_pm_channel_pause(bce);
+ if (status)
+ pr_warn("t2bce_core: shutdown mailbox quiesce failed: %d\n", status);
+ }
+
+ mutex_unlock(&bce->pm_lock);
+}
+
+static int bce_pm_suspend_fallback_no_state(struct t2bce_device *bce)
+{
+ pr_debug("t2bce_core: suspend: forcing SLEEP_NO_STATE (no reply expected)\n");
+ if (bce_mailbox_send_no_reply_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0))) {
+ pr_err("t2bce_core: suspend: SLEEP_NO_STATE send failed\n");
+ return -EIO;
+ }
+ return 0;
+}
+
+static int bce_pm_suspend_try_state(struct t2bce_device *bce)
+{
+ int status;
+ u64 resp;
+
+ /* Try the stateful bridgeOS path first; a reject falls back to no-state. */
+ bce->stateful_suspend_valid = false;
+
+ if (!bce->saved_data_dma_ptr || !bce->saved_data_dma_addr || !bce->saved_data_dma_size) {
+ pr_err("t2bce_core: suspend failed (persistent state buffer missing)\n");
+ return -ENOMEM;
+ }
+
+ BUG_ON((bce->saved_data_dma_addr % 4096) != 0);
+ status = bce_mailbox_send_locked(&bce->mbox,
+ BCE_MB_MSG(BCE_MB_SAVE_STATE_AND_SLEEP,
+ (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)),
+ &resp);
+ if (status) {
+ pr_err("t2bce_core: suspend failed (mailbox send)\n");
+ return status;
+ }
+
+ if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_RESTORE_STATE_COMPLETE) {
+ pr_debug("t2bce_core: suspend: remote response: restore state saved\n");
+ bce->stateful_suspend_valid = true;
+ return 0;
+ }
+
+ if (BCE_MB_TYPE(resp) == BCE_MB_SAVE_STATE_AND_SLEEP_REJECTED) {
+ pr_err("t2bce_core: remote rejected stateful suspend payload\n");
+ return -EAGAIN;
+ }
+ return -EINVAL;
+}
+
+static int bce_pm_resume_no_state(struct t2bce_device *bce)
+{
+ int status;
+ u64 resp;
+
+ if ((status = bce_mailbox_send_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &resp))) {
+ pr_err("t2bce_core: resume with no state failed (mailbox send)\n");
+ return status;
+ }
+ if (BCE_MB_TYPE(resp) != BCE_MB_RESTORE_NO_STATE) {
+ pr_err("t2bce_core: resume with no state failed (invalid device response)\n");
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static int bce_pm_resume_stateful(struct t2bce_device *bce)
+{
+ int status;
+ u64 resp;
+
+ if ((status = bce_mailbox_send_locked(&bce->mbox, BCE_MB_MSG(BCE_MB_RESTORE_STATE_AND_WAKE,
+ (bce->saved_data_dma_addr & ~(4096LLU - 1)) | (bce->saved_data_dma_size / 4096)), &resp))) {
+ pr_err("t2bce_core: resume with state failed (mailbox send)\n");
+ return status;
+ }
+ if (BCE_MB_TYPE(resp) != BCE_MB_SAVE_RESTORE_STATE_COMPLETE) {
+ pr_err("t2bce_core: resume with state failed (invalid device response)\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static int t2bce_suspend(struct device *dev)
+{
+ struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev));
+ int status;
+
+ pr_debug("t2bce_core: suspend: entry\n");
+ mutex_lock(&bce->pm_lock);
+
+ bce->stateful_suspend_valid = false;
+ bce->no_state_fallback = false;
+ bce->no_state_resume = false;
+ t2bce_core_clients_pm_reset(bce);
+ t2bce_core_clients_pm_prepare(bce);
+
+ status = bce_pm_suspend_prepare(bce);
+ if (status)
+ goto out_unlock;
+
+ if (!bce_stateful_supported(bce)) {
+ /* No-state resume tears down and rebuilds clients that cannot preserve firmware state. */
+ t2bce_core_clients_pm_prepare_no_state(bce);
+ status = bce_pm_suspend_fallback_no_state(bce);
+ if (!status) {
+ bce->no_state_fallback = true;
+ bce->no_state_resume = true;
+ t2bce_core_clients_pm_mark_no_state_resume(bce);
+ } else {
+ bce_pm_suspend_abort(bce);
+ }
+ goto out_unlock;
+ }
+
+ status = bce_pm_suspend_try_state(bce);
+ if (!status)
+ goto out_unlock;
+
+ if (status != -EAGAIN) {
+ bce_pm_suspend_abort(bce);
+ goto out_unlock;
+ }
+
+ /* bridgeOS can reject state save; fall back to the no-state path then. */
+ pr_debug("t2bce_core: suspend: stateful path not ready, falling back to no-state\n");
+ t2bce_core_clients_pm_prepare_no_state(bce);
+ status = bce_pm_suspend_fallback_no_state(bce);
+ if (!status) {
+ bce->no_state_fallback = true;
+ bce->no_state_resume = true;
+ t2bce_core_clients_pm_mark_no_state_resume(bce);
+ } else {
+ bce_pm_suspend_abort(bce);
+ }
+
+out_unlock:
+ mutex_unlock(&bce->pm_lock);
+ pr_info("t2bce_core: suspend: exit status=%d stateful_valid=%d no_state_resume=%d no_state_fallback=%d\n",
+ status, bce->stateful_suspend_valid, bce->no_state_resume, bce->no_state_fallback);
+ return status;
+}
+
+static int t2bce_resume(struct device *dev)
+{
+ struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev));
+ int status;
+ bool used_stateful;
+
+ pr_debug("t2bce_core: resume: entry\n");
+ mutex_lock(&bce->pm_lock);
+
+ pci_set_master(bce->pci);
+ pci_set_master(bce->pci0);
+
+ /* Resume follows the suspend result, not a preselected policy. */
+ used_stateful = bce_stateful_supported(bce) && bce->stateful_suspend_valid;
+ pr_debug("t2bce_core: resume path: %s\n", used_stateful ? "stateful" : "no-state");
+ if (used_stateful)
+ status = bce_pm_resume_stateful(bce);
+ else
+ status = bce_pm_resume_no_state(bce);
+ if (status)
+ goto out_unlock;
+
+ if (used_stateful)
+ bce->stateful_suspend_valid = false;
+
+ bce_pm_resume_finish(bce);
+
+out_unlock:
+ mutex_unlock(&bce->pm_lock);
+ pr_info("t2bce_core: resume: exit status=%d path=%s stateful_valid=%d no_state_resume=%d no_state_fallback=%d\n",
+ status, used_stateful ? "stateful" : "no-state",
+ bce->stateful_suspend_valid, bce->no_state_resume, bce->no_state_fallback);
+ return status;
+}
+
+static int t2bce_prepare(struct device *dev)
+{
+ /*
+ * Force PCI PM to run our real suspend/resume callbacks instead of
+ * short-cutting an "already runtime suspended" device through complete().
+ * The T2 mailbox save/restore handshake lives in those callbacks.
+ */
+ return 0;
+}
+
+static void t2bce_complete(struct device *dev)
+{
+ struct t2bce_device *bce = pci_get_drvdata(to_pci_dev(dev));
+
+ pr_debug("t2bce_core: complete: entry no_state_fallback=%d no_state_resume=%d\n",
+ bce->no_state_fallback, bce->no_state_resume);
+ if (bce->no_state_fallback && t2bce_core_clients_pm_has_no_state_resume(bce)) {
+ t2bce_core_clients_pm_complete(bce);
+ bce->no_state_fallback = false;
+ }
+
+ t2bce_core_notify_resume_complete(bce);
+ bce->no_state_resume = false;
+ pr_debug("t2bce_core: complete: exit\n");
+}
+
+static struct pci_device_id t2bce_ids[ ] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x1801) },
+ { 0, },
+};
+
+MODULE_DEVICE_TABLE(pci, t2bce_ids);
+
+struct dev_pm_ops t2bce_pci_driver_pm = {
+ .prepare = t2bce_prepare,
+ .suspend = t2bce_suspend,
+ .resume = t2bce_resume,
+ .complete = t2bce_complete
+};
+struct pci_driver t2bce_pci_driver = {
+ .name = "t2bce_core",
+ .id_table = t2bce_ids,
+ .probe = t2bce_probe,
+ .remove = t2bce_remove,
+ .shutdown = t2bce_shutdown,
+ .driver = {
+ .pm = &t2bce_pci_driver_pm
+ }
+};
+
+
+static int __init t2bce_module_init(void)
+{
+ int result;
+
+ if ((result = alloc_chrdev_region(&bce_chrdev, 0, 1, "t2bce_core")))
+ goto fail_chrdev;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0)
+ bce_class = class_create(THIS_MODULE, "t2bce_core");
+#else
+ bce_class = class_create("t2bce_core");
+#endif
+ if (IS_ERR(bce_class)) {
+ result = PTR_ERR(bce_class);
+ goto fail_class;
+ }
+ result = pci_register_driver(&t2bce_pci_driver);
+ if (result)
+ goto fail_drv;
+
+ pr_info("t2bce_core: module initialized\n");
+ return 0;
+
+fail_drv:
+ pci_unregister_driver(&t2bce_pci_driver);
+fail_class:
+ class_destroy(bce_class);
+fail_chrdev:
+ unregister_chrdev_region(bce_chrdev, 1);
+ if (!result)
+ result = -EINVAL;
+ pr_info("t2bce_core: module init failed status=%d\n", result);
+ return result;
+}
+static void __exit t2bce_module_exit(void)
+{
+ pci_unregister_driver(&t2bce_pci_driver);
+
+ class_destroy(bce_class);
+ unregister_chrdev_region(bce_chrdev, 1);
+ pr_info("t2bce_core: module exited\n");
+}
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("André Eikmeyer <andre.eikmeyer@gmail.com>");
+MODULE_DESCRIPTION("T2 BCE core driver");
+MODULE_VERSION("0.06");
+MODULE_SOFTDEP("post: t2bce_vhci");
+module_init(t2bce_module_init);
+module_exit(t2bce_module_exit);
diff --git a/drivers/staging/t2bce/t2bce_core/transport.c b/drivers/staging/t2bce/t2bce_core/transport.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_core/transport.c
@@ -0,0 +1,408 @@
+#include "t2bce_core_transport.h"
+#include "t2bce.h"
+
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+struct t2bce_core_client {
+ struct t2bce_device *bce;
+ struct device *dev;
+ struct device_link *link;
+ struct list_head list;
+ t2bce_core_client_resume_callback resume_complete;
+ void *resume_complete_userdata;
+ struct t2bce_core_client_pm_ops pm_ops;
+ void *pm_userdata;
+};
+
+struct t2bce_sq_ctx {
+ t2bce_core_sq_completion completion;
+ void *userdata;
+};
+
+static struct bce_queue_cq *to_bce_cq(struct t2bce_core_queue_cq *cq)
+{
+ return (struct bce_queue_cq *) cq;
+}
+
+static struct t2bce_core_queue_cq *to_t2bce_cq(struct bce_queue_cq *cq)
+{
+ return (struct t2bce_core_queue_cq *) cq;
+}
+
+static struct bce_queue_sq *to_bce_sq(struct t2bce_core_queue_sq *sq)
+{
+ return (struct bce_queue_sq *) sq;
+}
+
+static struct t2bce_core_queue_sq *to_t2bce_sq(struct bce_queue_sq *sq)
+{
+ return (struct t2bce_core_queue_sq *) sq;
+}
+
+static void t2bce_sq_completion_adapter(struct bce_queue_sq *sq)
+{
+ struct t2bce_sq_ctx *ctx = sq->userdata;
+
+ ctx->completion(to_t2bce_sq(sq));
+}
+
+struct t2bce_core_client *t2bce_core_client_get(struct device *dev)
+{
+ struct t2bce_device *bce = global_bce;
+ struct t2bce_core_client *client;
+
+ if (!bce)
+ return ERR_PTR(-EPROBE_DEFER);
+
+ if (bce->is_being_removed)
+ return ERR_PTR(-ENODEV);
+
+ client = kzalloc(sizeof(*client), GFP_KERNEL);
+ if (!client)
+ return ERR_PTR(-ENOMEM);
+
+ client->bce = bce;
+ client->dev = dev;
+ client->link = device_link_add(dev, &bce->pci->dev,
+ DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_CONSUMER);
+ if (!client->link) {
+ kfree(client);
+ return ERR_PTR(-ENODEV);
+ }
+
+ mutex_lock(&bce->clients_lock);
+ list_add_tail_rcu(&client->list, &bce->clients);
+ mutex_unlock(&bce->clients_lock);
+
+ return client;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_client_get);
+
+void t2bce_core_client_put(struct t2bce_core_client *client)
+{
+ if (!client)
+ return;
+
+ mutex_lock(&client->bce->clients_lock);
+ list_del_rcu(&client->list);
+ mutex_unlock(&client->bce->clients_lock);
+
+ synchronize_srcu(&client->bce->clients_srcu);
+
+ if (client->link)
+ device_link_del(client->link);
+ kfree(client);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_client_put);
+
+struct device *t2bce_core_device_get(void)
+{
+ struct t2bce_device *bce = global_bce;
+
+ if (!bce)
+ return ERR_PTR(-EPROBE_DEFER);
+
+ if (bce->is_being_removed || !bce->dev)
+ return ERR_PTR(-ENODEV);
+
+ return get_device(bce->dev);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_device_get);
+
+void t2bce_core_device_put(struct device *dev)
+{
+ if (dev)
+ put_device(dev);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_device_put);
+
+struct device *t2bce_core_client_dma_dev(struct t2bce_core_client *client)
+{
+ return &client->bce->pci->dev;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_client_dma_dev);
+
+bool t2bce_core_client_no_state_resume(struct t2bce_core_client *client)
+{
+ return client->bce->no_state_resume;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_client_no_state_resume);
+
+void t2bce_core_client_set_resume_complete_callback(struct t2bce_core_client *client,
+ t2bce_core_client_resume_callback callback, void *userdata)
+{
+ if (!callback) {
+ smp_store_release(&client->resume_complete, NULL);
+ WRITE_ONCE(client->resume_complete_userdata, NULL);
+ return;
+ }
+
+ WRITE_ONCE(client->resume_complete_userdata, userdata);
+ smp_store_release(&client->resume_complete, callback);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_client_set_resume_complete_callback);
+
+void t2bce_core_client_set_pm_ops(struct t2bce_core_client *client,
+ const struct t2bce_core_client_pm_ops *ops, void *userdata)
+{
+ if (!ops) {
+ memset(&client->pm_ops, 0, sizeof(client->pm_ops));
+ WRITE_ONCE(client->pm_userdata, NULL);
+ return;
+ }
+
+ WRITE_ONCE(client->pm_userdata, userdata);
+ client->pm_ops = *ops;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_client_set_pm_ops);
+
+void t2bce_core_clients_shutdown(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.shutdown)
+ client->pm_ops.shutdown(READ_ONCE(client->pm_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+void t2bce_core_clients_pm_reset(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.pm_reset)
+ client->pm_ops.pm_reset(READ_ONCE(client->pm_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+void t2bce_core_clients_pm_prepare(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.pm_prepare)
+ client->pm_ops.pm_prepare(READ_ONCE(client->pm_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+void t2bce_core_clients_pm_prepare_no_state(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.pm_prepare_no_state)
+ client->pm_ops.pm_prepare_no_state(READ_ONCE(client->pm_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+void t2bce_core_clients_pm_mark_no_state_resume(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.pm_mark_no_state_resume)
+ client->pm_ops.pm_mark_no_state_resume(READ_ONCE(client->pm_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+bool t2bce_core_clients_pm_has_no_state_resume(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ bool ret = false;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.pm_is_no_state_resume &&
+ client->pm_ops.pm_is_no_state_resume(READ_ONCE(client->pm_userdata))) {
+ ret = true;
+ break;
+ }
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+ return ret;
+}
+
+void t2bce_core_clients_pm_complete(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ if (client->pm_ops.pm_complete)
+ client->pm_ops.pm_complete(READ_ONCE(client->pm_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+void t2bce_core_notify_resume_complete(struct t2bce_device *bce)
+{
+ struct t2bce_core_client *client;
+ int srcu_idx;
+
+ srcu_idx = srcu_read_lock(&bce->clients_srcu);
+ list_for_each_entry_srcu(client, &bce->clients, list,
+ srcu_read_lock_held(&bce->clients_srcu)) {
+ t2bce_core_client_resume_callback callback = smp_load_acquire(&client->resume_complete);
+
+ if (callback)
+ callback(READ_ONCE(client->resume_complete_userdata));
+ }
+ srcu_read_unlock(&bce->clients_srcu, srcu_idx);
+}
+
+struct t2bce_core_queue_cq *t2bce_core_create_cq(struct t2bce_core_client *client, u32 el_count)
+{
+ return to_t2bce_cq(t2bce_dma_create_cq(&client->bce->dma, el_count));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_create_cq);
+
+void t2bce_core_destroy_cq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq)
+{
+ t2bce_dma_destroy_cq(&client->bce->dma, to_bce_cq(cq));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_destroy_cq);
+
+struct t2bce_core_queue_sq *t2bce_core_create_sq(struct t2bce_core_client *client, struct t2bce_core_queue_cq *cq,
+ const char *name, u32 el_count, enum dma_data_direction direction,
+ t2bce_core_sq_completion compl, void *userdata)
+{
+ struct t2bce_sq_ctx *ctx;
+ struct bce_queue_sq *sq;
+
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return NULL;
+
+ ctx->completion = compl;
+ ctx->userdata = userdata;
+
+ sq = t2bce_dma_create_sq(&client->bce->dma, to_bce_cq(cq), name, el_count, direction,
+ t2bce_sq_completion_adapter, ctx);
+ if (!sq) {
+ kfree(ctx);
+ return NULL;
+ }
+
+ return to_t2bce_sq(sq);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_create_sq);
+
+void t2bce_core_destroy_sq(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq)
+{
+ struct bce_queue_sq *bce_sq = to_bce_sq(sq);
+ struct t2bce_sq_ctx *ctx = bce_sq->userdata;
+
+ t2bce_dma_destroy_sq(&client->bce->dma, bce_sq);
+ kfree(ctx);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_destroy_sq);
+
+void *t2bce_core_queue_sq_userdata(struct t2bce_core_queue_sq *sq)
+{
+ struct t2bce_sq_ctx *ctx = to_bce_sq(sq)->userdata;
+
+ return ctx->userdata;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_userdata);
+
+int t2bce_core_reserve_submission(struct t2bce_core_queue_sq *sq, unsigned long *timeout)
+{
+ return t2bce_dma_reserve_submission(to_bce_sq(sq), timeout);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_reserve_submission);
+
+void t2bce_core_cancel_submission_reservation(struct t2bce_core_queue_sq *sq)
+{
+ t2bce_dma_cancel_submission_reservation(to_bce_sq(sq));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_cancel_submission_reservation);
+
+void t2bce_core_set_next_submission_single(struct t2bce_core_queue_sq *sq, dma_addr_t addr, size_t size)
+{
+ t2bce_dma_set_next_submission_single(to_bce_sq(sq), addr, size);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_set_next_submission_single);
+
+void t2bce_core_set_next_submission_segment_list(struct t2bce_core_queue_sq *sq,
+ dma_addr_t segl_addr, size_t segl_size)
+{
+ t2bce_dma_set_next_submission_segment_list(to_bce_sq(sq), segl_addr, segl_size);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_set_next_submission_segment_list);
+
+void t2bce_core_submit_to_device(struct t2bce_core_queue_sq *sq)
+{
+ t2bce_dma_submit_to_device(to_bce_sq(sq));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_submit_to_device);
+
+void t2bce_core_notify_submission_complete(struct t2bce_core_queue_sq *sq)
+{
+ t2bce_dma_notify_submission_complete(to_bce_sq(sq));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_notify_submission_complete);
+
+struct t2bce_core_sq_completion_data *t2bce_core_next_completion(struct t2bce_core_queue_sq *sq)
+{
+ return (struct t2bce_core_sq_completion_data *) t2bce_dma_next_completion(to_bce_sq(sq));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_next_completion);
+
+u32 t2bce_core_queue_sq_head(struct t2bce_core_queue_sq *sq)
+{
+ return to_bce_sq(sq)->head;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_head);
+
+u32 t2bce_core_queue_sq_tail(struct t2bce_core_queue_sq *sq)
+{
+ return to_bce_sq(sq)->tail;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_tail);
+
+u32 t2bce_core_queue_sq_available(struct t2bce_core_queue_sq *sq)
+{
+ return atomic_read(&to_bce_sq(sq)->available_commands);
+}
+EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_available);
+
+u32 t2bce_core_queue_sq_capacity(struct t2bce_core_queue_sq *sq)
+{
+ return to_bce_sq(sq)->el_count;
+}
+EXPORT_SYMBOL_GPL(t2bce_core_queue_sq_capacity);
+
+int t2bce_core_flush_queue(struct t2bce_core_client *client, struct t2bce_core_queue_sq *sq)
+{
+ return t2bce_dma_flush_sq(&client->bce->dma, to_bce_sq(sq));
+}
+EXPORT_SYMBOL_GPL(t2bce_core_flush_queue);
diff --git a/drivers/staging/t2bce/t2bce_dma/Makefile b/drivers/staging/t2bce/t2bce_dma/Makefile
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_dma/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_T2BCE_DMA) += t2bce_dma.o
+t2bce_dma-y := queue.o
+
+ccflags-y += -DWITHOUT_NVME_PATCH
+ccflags-y += -I$(src)/include
diff --git a/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h b/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_dma/include/t2bce_dma_queue.h
@@ -0,0 +1,182 @@
+#ifndef T2BCE_DMA_QUEUE_H
+#define T2BCE_DMA_QUEUE_H
+
+#include <linux/completion.h>
+#include <linux/pci.h>
+
+#define BCE_CMD_SIZE 0x40
+#define BCE_MAX_QUEUE_COUNT 0x100
+
+#define BCE_QUEUE_USER_MIN 2
+#define BCE_QUEUE_USER_MAX (BCE_MAX_QUEUE_COUNT - 1)
+
+struct t2bce_dma_segment {
+ u64 addr;
+ u64 length;
+};
+
+struct bce_queue_memcfg;
+
+struct t2bce_dma_engine_ops {
+ int (*register_queue)(void *userdata, struct bce_queue_memcfg *cfg,
+ const char *name, bool isdirout);
+ int (*unregister_queue)(void *userdata, u16 qid);
+ int (*flush_queue)(void *userdata, u16 qid);
+};
+
+enum bce_queue_type {
+ BCE_QUEUE_CQ, BCE_QUEUE_SQ
+};
+struct bce_queue {
+ int qid;
+ int type;
+};
+struct bce_queue_cq {
+ int qid;
+ int type;
+ u32 el_count;
+ dma_addr_t dma_handle;
+ void *data;
+
+ u32 index;
+};
+struct bce_queue_sq;
+typedef void (*bce_sq_completion)(struct bce_queue_sq *q);
+struct bce_sq_completion_data {
+ u32 status;
+ u64 data_size;
+ u64 result;
+};
+struct bce_queue_sq {
+ int qid;
+ int type;
+ u32 el_size;
+ u32 el_count;
+ dma_addr_t dma_handle;
+ void *data;
+ void *userdata;
+ void __iomem *reg_mem_dma;
+
+ atomic_t available_commands;
+ struct completion available_command_completion;
+ atomic_t available_command_completion_waiting_count;
+ u32 head, tail;
+
+ u32 completion_cidx, completion_tail;
+ struct bce_sq_completion_data *completion_data;
+ bool has_pending_completions;
+ bce_sq_completion completion;
+};
+
+struct bce_queue_cmdq_result_el {
+ struct completion cmpl;
+ u32 status;
+ u64 result;
+ u32 slot;
+ u32 generation;
+};
+struct bce_queue_cmdq {
+ struct bce_queue_sq *sq;
+ struct spinlock lck;
+ struct bce_queue_cmdq_result_el **tres;
+ u32 *slot_gen;
+};
+
+struct t2bce_dma_engine {
+ struct device *dma_dev;
+ const struct t2bce_dma_engine_ops *ops;
+ void *ops_userdata;
+ void __iomem *reg_mem_dma;
+ struct bce_queue *queues[BCE_MAX_QUEUE_COUNT];
+ struct spinlock queues_lock;
+ struct ida queue_ida;
+ struct bce_queue_cq *cmd_cq;
+ struct bce_queue_cmdq *cmd_cmdq;
+ struct bce_queue_sq *int_sq_list[BCE_MAX_QUEUE_COUNT];
+ bool is_being_removed;
+};
+
+struct bce_queue_memcfg {
+ u16 qid;
+ u16 el_count;
+ u16 vector_or_cq;
+ u16 _pad;
+ u64 addr;
+ u64 length;
+};
+
+enum bce_qe_completion_status {
+ BCE_COMPLETION_SUCCESS = 0,
+ BCE_COMPLETION_ERROR = 1,
+ BCE_COMPLETION_ABORTED = 2,
+ BCE_COMPLETION_NO_SPACE = 3,
+ BCE_COMPLETION_OVERRUN = 4
+};
+enum bce_qe_completion_flags {
+ BCE_COMPLETION_FLAG_PENDING = 0x8000
+};
+struct bce_qe_completion {
+ u64 result;
+ u64 data_size;
+ u16 qid;
+ u16 completion_index;
+ u16 status; // bce_qe_completion_status
+ u16 flags; // bce_qe_completion_flags
+};
+
+static __always_inline void *t2bce_dma_sq_element(struct bce_queue_sq *q, int i) {
+ return (void *) ((u8 *) q->data + q->el_size * i);
+}
+static __always_inline void *t2bce_dma_cq_element(struct bce_queue_cq *q, int i) {
+ return (void *) ((struct bce_qe_completion *) q->data + i);
+}
+
+static __always_inline struct bce_sq_completion_data *t2bce_dma_next_completion(struct bce_queue_sq *sq) {
+ struct bce_sq_completion_data *res;
+ rmb();
+ if (sq->completion_cidx == sq->completion_tail)
+ return NULL;
+ res = &sq->completion_data[sq->completion_cidx];
+ sq->completion_cidx = (sq->completion_cidx + 1) % sq->el_count;
+ return res;
+}
+
+struct bce_queue_cq *t2bce_dma_alloc_cq(struct t2bce_dma_engine *dma, int qid, u32 el_count);
+void t2bce_dma_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg);
+void t2bce_dma_free_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq);
+void t2bce_dma_handle_cq_completions_locked(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, size_t *ce);
+void t2bce_dma_dispatch_pending_sq_completions(struct t2bce_dma_engine *dma, size_t ce);
+
+struct bce_queue_sq *t2bce_dma_alloc_sq(struct t2bce_dma_engine *dma, int qid, u32 el_size, u32 el_count,
+ bce_sq_completion compl, void *userdata);
+void t2bce_dma_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg);
+void t2bce_dma_free_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq);
+int t2bce_dma_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout);
+void t2bce_dma_cancel_submission_reservation(struct bce_queue_sq *sq);
+void t2bce_dma_submit_to_device(struct bce_queue_sq *sq);
+void t2bce_dma_notify_submission_complete(struct bce_queue_sq *sq);
+
+void t2bce_dma_set_next_submission_single(struct bce_queue_sq *sq, dma_addr_t addr, size_t size);
+void t2bce_dma_set_next_submission_segment_list(struct bce_queue_sq *sq,
+ dma_addr_t segl_addr, size_t segl_size);
+
+struct bce_queue_cmdq *t2bce_dma_alloc_cmdq(struct t2bce_dma_engine *dma, int qid, u32 el_count);
+void t2bce_dma_free_cmdq(struct t2bce_dma_engine *dma, struct bce_queue_cmdq *cmdq);
+
+u32 t2bce_dma_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout);
+u32 t2bce_dma_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid);
+u32 t2bce_dma_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid);
+
+
+/* User API - Creates and registers the queue */
+
+struct bce_queue_cq *t2bce_dma_create_cq(struct t2bce_dma_engine *dma, u32 el_count);
+struct bce_queue_sq *t2bce_dma_create_sq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, u32 el_count,
+ int direction, bce_sq_completion compl, void *userdata);
+struct bce_queue_sq *t2bce_dma_create_sq_with_flags(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name,
+ u32 el_count, u16 flags, bce_sq_completion compl, void *userdata);
+void t2bce_dma_destroy_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq);
+void t2bce_dma_destroy_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq);
+int t2bce_dma_flush_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq);
+
+#endif
diff --git a/drivers/staging/t2bce/t2bce_dma/queue.c b/drivers/staging/t2bce/t2bce_dma/queue.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_dma/queue.c
@@ -0,0 +1,625 @@
+#include "t2bce_dma_queue.h"
+#include <linux/export.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/version.h>
+
+#define REG_DOORBELL_BASE 0x44000
+
+struct bce_qe_submission {
+ u64 length;
+ u64 addr;
+
+ u64 segl_addr;
+ u64 segl_length;
+};
+
+enum bce_submission_type {
+ BCE_SUBMISSION_SINGLE,
+ BCE_SUBMISSION_SEGMENT_LIST,
+};
+
+struct bce_submission {
+ enum bce_submission_type type;
+ union {
+ struct {
+ dma_addr_t addr;
+ size_t size;
+ } single;
+ struct {
+ dma_addr_t addr;
+ size_t size;
+ } segment_list;
+ };
+};
+
+enum bce_cmdq_command {
+ BCE_CMD_REGISTER_MEMORY_QUEUE = 0x20,
+ BCE_CMD_UNREGISTER_MEMORY_QUEUE = 0x30,
+ BCE_CMD_FLUSH_MEMORY_QUEUE = 0x40,
+ BCE_CMD_SET_MEMORY_QUEUE_PROPERTY = 0x50
+};
+struct bce_cmdq_simple_memory_queue_cmd {
+ u16 cmd; // bce_cmdq_command
+ u16 flags;
+ u16 qid;
+};
+struct bce_cmdq_register_memory_queue_cmd {
+ u16 cmd; // bce_cmdq_command
+ u16 flags;
+ u16 qid;
+ u16 _pad;
+ u16 el_count;
+ u16 vector_or_cq;
+ u16 _pad2;
+ u16 name_len;
+ char name[0x20];
+ u64 addr;
+ u64 length;
+};
+
+struct bce_queue_cq *t2bce_dma_alloc_cq(struct t2bce_dma_engine *dma, int qid, u32 el_count)
+{
+ struct bce_queue_cq *q;
+ q = kzalloc(sizeof(struct bce_queue_cq), GFP_KERNEL);
+ if (!q)
+ return NULL;
+ q->qid = qid;
+ q->type = BCE_QUEUE_CQ;
+ q->el_count = el_count;
+ q->data = dma_alloc_coherent(dma->dma_dev, el_count * sizeof(struct bce_qe_completion),
+ &q->dma_handle, GFP_KERNEL);
+ if (!q->data) {
+ pr_err("DMA queue memory alloc failed\n");
+ kfree(q);
+ return NULL;
+ }
+ return q;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_alloc_cq);
+
+void t2bce_dma_get_cq_memcfg(struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg)
+{
+ cfg->qid = (u16) cq->qid;
+ cfg->el_count = (u16) cq->el_count;
+ cfg->vector_or_cq = 0;
+ cfg->_pad = 0;
+ cfg->addr = cq->dma_handle;
+ cfg->length = cq->el_count * sizeof(struct bce_qe_completion);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_get_cq_memcfg);
+
+void t2bce_dma_free_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq)
+{
+ dma_free_coherent(dma->dma_dev, cq->el_count * sizeof(struct bce_qe_completion), cq->data, cq->dma_handle);
+ kfree(cq);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_free_cq);
+
+static void t2bce_dma_handle_cq_completion(struct t2bce_dma_engine *dma, struct bce_qe_completion *e, size_t *ce)
+{
+ struct bce_queue *target;
+ struct bce_queue_sq *target_sq;
+ struct bce_sq_completion_data *cmpl;
+ if (e->qid >= BCE_MAX_QUEUE_COUNT) {
+ pr_err("Device sent a response for qid (%u) >= BCE_MAX_QUEUE_COUNT\n", e->qid);
+ return;
+ }
+ target = dma->queues[e->qid];
+ if (!target || target->type != BCE_QUEUE_SQ) {
+ pr_err("Device sent a response for qid (%u), which does not exist\n", e->qid);
+ return;
+ }
+ target_sq = (struct bce_queue_sq *) target;
+ if (target_sq->completion_tail != e->completion_index) {
+ pr_err("Completion index mismatch; this is likely going to make this driver unusable\n");
+ return;
+ }
+ if (!target_sq->has_pending_completions) {
+ target_sq->has_pending_completions = true;
+ dma->int_sq_list[(*ce)++] = target_sq;
+ }
+ cmpl = &target_sq->completion_data[e->completion_index];
+ cmpl->status = e->status;
+ cmpl->data_size = e->data_size;
+ cmpl->result = e->result;
+ wmb();
+ target_sq->completion_tail = (target_sq->completion_tail + 1) % target_sq->el_count;
+}
+
+void t2bce_dma_handle_cq_completions_locked(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, size_t *ce)
+{
+ struct bce_qe_completion *e;
+ e = t2bce_dma_cq_element(cq, cq->index);
+ if (!(e->flags & BCE_COMPLETION_FLAG_PENDING))
+ return;
+ mb();
+ while (true) {
+ e = t2bce_dma_cq_element(cq, cq->index);
+ if (!(e->flags & BCE_COMPLETION_FLAG_PENDING))
+ break;
+ pr_debug("t2bce_dma: compl: %i: %i %llx %llx", e->qid, e->status, e->data_size, e->result);
+ t2bce_dma_handle_cq_completion(dma, e, ce);
+ e->flags = 0;
+ cq->index = (cq->index + 1) % cq->el_count;
+ }
+ mb();
+ iowrite32(cq->index, (u32 *) ((u8 *) dma->reg_mem_dma + REG_DOORBELL_BASE) + cq->qid);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_handle_cq_completions_locked);
+
+void t2bce_dma_dispatch_pending_sq_completions(struct t2bce_dma_engine *dma, size_t ce)
+{
+ struct bce_queue_sq *sq;
+
+ while (ce) {
+ --ce;
+ sq = dma->int_sq_list[ce];
+ sq->completion(sq);
+ sq->has_pending_completions = false;
+ }
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_dispatch_pending_sq_completions);
+
+
+struct bce_queue_sq *t2bce_dma_alloc_sq(struct t2bce_dma_engine *dma, int qid, u32 el_size, u32 el_count,
+ bce_sq_completion compl, void *userdata)
+{
+ struct bce_queue_sq *q;
+ q = kzalloc(sizeof(struct bce_queue_sq), GFP_KERNEL);
+ if (!q)
+ return NULL;
+ q->qid = qid;
+ q->type = BCE_QUEUE_SQ;
+ q->el_size = el_size;
+ q->el_count = el_count;
+ q->data = dma_alloc_coherent(dma->dma_dev, el_count * el_size,
+ &q->dma_handle, GFP_KERNEL);
+ q->completion = compl;
+ q->userdata = userdata;
+ q->completion_data = kzalloc(sizeof(struct bce_sq_completion_data) * el_count, GFP_KERNEL);
+ q->reg_mem_dma = dma->reg_mem_dma;
+ atomic_set(&q->available_commands, el_count - 1);
+ init_completion(&q->available_command_completion);
+ atomic_set(&q->available_command_completion_waiting_count, 0);
+ if (!q->data || !q->completion_data) {
+ pr_err("DMA queue memory alloc failed\n");
+ if (q->data)
+ dma_free_coherent(dma->dma_dev, el_count * el_size, q->data, q->dma_handle);
+ kfree(q->completion_data);
+ kfree(q);
+ return NULL;
+ }
+ return q;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_alloc_sq);
+
+void t2bce_dma_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq, struct bce_queue_memcfg *cfg)
+{
+ cfg->qid = (u16) sq->qid;
+ cfg->el_count = (u16) sq->el_count;
+ cfg->vector_or_cq = (u16) cq->qid;
+ cfg->_pad = 0;
+ cfg->addr = sq->dma_handle;
+ cfg->length = sq->el_count * sq->el_size;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_get_sq_memcfg);
+
+void t2bce_dma_free_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq)
+{
+ dma_free_coherent(dma->dma_dev, sq->el_count * sq->el_size, sq->data, sq->dma_handle);
+ kfree(sq->completion_data);
+ kfree(sq);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_free_sq);
+
+int t2bce_dma_reserve_submission(struct bce_queue_sq *sq, unsigned long *timeout)
+{
+ while (atomic_dec_if_positive(&sq->available_commands) < 0) {
+ if (!timeout || !*timeout)
+ return -EAGAIN;
+ atomic_inc(&sq->available_command_completion_waiting_count);
+ *timeout = wait_for_completion_timeout(&sq->available_command_completion, *timeout);
+ if (!*timeout) {
+ if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) < 0)
+ try_wait_for_completion(&sq->available_command_completion); /* consume the pending completion */
+ }
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_reserve_submission);
+
+void t2bce_dma_cancel_submission_reservation(struct bce_queue_sq *sq)
+{
+ atomic_inc(&sq->available_commands);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_cancel_submission_reservation);
+
+static void *t2bce_dma_next_submission(struct bce_queue_sq *sq)
+{
+ void *ret = t2bce_dma_sq_element(sq, sq->tail);
+ sq->tail = (sq->tail + 1) % sq->el_count;
+ return ret;
+}
+
+void t2bce_dma_submit_to_device(struct bce_queue_sq *sq)
+{
+ mb();
+ iowrite32(sq->tail, (u32 *) ((u8 *) sq->reg_mem_dma + REG_DOORBELL_BASE) + sq->qid);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_submit_to_device);
+
+void t2bce_dma_notify_submission_complete(struct bce_queue_sq *sq)
+{
+ sq->head = (sq->head + 1) % sq->el_count;
+ atomic_inc(&sq->available_commands);
+ if (atomic_dec_if_positive(&sq->available_command_completion_waiting_count) >= 0) {
+ complete(&sq->available_command_completion);
+ }
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_notify_submission_complete);
+
+static void t2bce_dma_write_submission(struct bce_qe_submission *element, const struct bce_submission *submission)
+{
+ switch (submission->type) {
+ case BCE_SUBMISSION_SINGLE:
+ element->addr = submission->single.addr;
+ element->length = submission->single.size;
+ element->segl_addr = 0;
+ element->segl_length = 0;
+ break;
+ case BCE_SUBMISSION_SEGMENT_LIST:
+ element->addr = 0;
+ element->length = 0;
+ element->segl_addr = submission->segment_list.addr;
+ element->segl_length = submission->segment_list.size;
+ break;
+ }
+}
+
+void t2bce_dma_set_next_submission_single(struct bce_queue_sq *sq, dma_addr_t addr, size_t size)
+{
+ struct bce_submission submission = {
+ .type = BCE_SUBMISSION_SINGLE,
+ .single = {
+ .addr = addr,
+ .size = size,
+ },
+ };
+
+ t2bce_dma_write_submission(t2bce_dma_next_submission(sq), &submission);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_set_next_submission_single);
+
+void t2bce_dma_set_next_submission_segment_list(struct bce_queue_sq *sq,
+ dma_addr_t segl_addr, size_t segl_size)
+{
+ struct bce_submission submission = {
+ .type = BCE_SUBMISSION_SEGMENT_LIST,
+ .segment_list = {
+ .addr = segl_addr,
+ .size = segl_size,
+ },
+ };
+
+ t2bce_dma_write_submission(t2bce_dma_next_submission(sq), &submission);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_set_next_submission_segment_list);
+
+static void t2bce_dma_cmdq_completion(struct bce_queue_sq *q);
+
+struct bce_queue_cmdq *t2bce_dma_alloc_cmdq(struct t2bce_dma_engine *dma, int qid, u32 el_count)
+{
+ struct bce_queue_cmdq *q;
+ q = kzalloc(sizeof(struct bce_queue_cmdq), GFP_KERNEL);
+ if (!q)
+ return NULL;
+ q->sq = t2bce_dma_alloc_sq(dma, qid, BCE_CMD_SIZE, el_count, t2bce_dma_cmdq_completion, q);
+ if (!q->sq) {
+ kfree(q);
+ return NULL;
+ }
+ spin_lock_init(&q->lck);
+ q->tres = kzalloc(sizeof(struct bce_queue_cmdq_result_el*) * el_count, GFP_KERNEL);
+ if (!q->tres) {
+ t2bce_dma_free_sq(dma, q->sq);
+ kfree(q);
+ return NULL;
+ }
+ q->slot_gen = kzalloc(sizeof(u32) * el_count, GFP_KERNEL);
+ if (!q->slot_gen) {
+ kfree(q->tres);
+ t2bce_dma_free_sq(dma, q->sq);
+ kfree(q);
+ return NULL;
+ }
+ return q;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_alloc_cmdq);
+
+void t2bce_dma_free_cmdq(struct t2bce_dma_engine *dma, struct bce_queue_cmdq *cmdq)
+{
+ t2bce_dma_free_sq(dma, cmdq->sq);
+ kfree(cmdq->slot_gen);
+ kfree(cmdq->tres);
+ kfree(cmdq);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_free_cmdq);
+
+void t2bce_dma_cmdq_completion(struct bce_queue_sq *q)
+{
+ struct bce_queue_cmdq_result_el *el;
+ struct bce_queue_cmdq *cmdq = q->userdata;
+ struct bce_sq_completion_data *result;
+
+ spin_lock(&cmdq->lck);
+ while ((result = t2bce_dma_next_completion(q))) {
+ el = cmdq->tres[cmdq->sq->head];
+ if (el && el->generation == cmdq->slot_gen[cmdq->sq->head]) {
+ el->result = result->result;
+ el->status = result->status;
+ mb();
+ complete(&el->cmpl);
+ }
+ cmdq->tres[cmdq->sq->head] = NULL;
+ t2bce_dma_notify_submission_complete(q);
+ }
+ spin_unlock(&cmdq->lck);
+}
+
+static __always_inline void *t2bce_dma_cmd_start(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res)
+{
+ void *ret;
+ unsigned long timeout;
+ init_completion(&res->cmpl);
+ mb();
+
+ timeout = msecs_to_jiffies(1000L * 60 * 5); /* wait for up to ~5 minutes */
+ if (t2bce_dma_reserve_submission(cmdq->sq, &timeout))
+ return NULL;
+
+ spin_lock(&cmdq->lck);
+ res->slot = cmdq->sq->tail;
+ res->generation = cmdq->slot_gen[cmdq->sq->tail];
+ cmdq->tres[cmdq->sq->tail] = res;
+ ret = t2bce_dma_next_submission(cmdq->sq);
+ return ret;
+}
+
+static __always_inline int t2bce_dma_cmd_finish(struct bce_queue_cmdq *cmdq, struct bce_queue_cmdq_result_el *res)
+{
+ t2bce_dma_submit_to_device(cmdq->sq);
+ spin_unlock(&cmdq->lck);
+
+ if (!wait_for_completion_timeout(&res->cmpl, msecs_to_jiffies(5000))) {
+ pr_err("t2bce_dma: command queue timeout (slot %u)\n", res->slot);
+ spin_lock(&cmdq->lck);
+ cmdq->tres[res->slot] = NULL;
+ cmdq->slot_gen[res->slot]++;
+ spin_unlock(&cmdq->lck);
+ t2bce_dma_notify_submission_complete(cmdq->sq);
+ return -ETIMEDOUT;
+ }
+ mb();
+ return 0;
+}
+
+u32 t2bce_dma_cmd_register_queue(struct bce_queue_cmdq *cmdq, struct bce_queue_memcfg *cfg, const char *name, bool isdirout)
+{
+ struct bce_queue_cmdq_result_el res;
+ struct bce_cmdq_register_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res);
+ if (!cmd)
+ return (u32) -1;
+ cmd->cmd = BCE_CMD_REGISTER_MEMORY_QUEUE;
+ cmd->flags = (u16) ((name ? 2 : 0) | (isdirout ? 1 : 0));
+ cmd->qid = cfg->qid;
+ cmd->el_count = cfg->el_count;
+ cmd->vector_or_cq = cfg->vector_or_cq;
+ memset(cmd->name, 0, sizeof(cmd->name));
+ if (name) {
+ cmd->name_len = (u16) min(strlen(name), (size_t) sizeof(cmd->name));
+ memcpy(cmd->name, name, cmd->name_len);
+ } else {
+ cmd->name_len = 0;
+ }
+ cmd->addr = cfg->addr;
+ cmd->length = cfg->length;
+
+ if (t2bce_dma_cmd_finish(cmdq, &res))
+ return (u32)-1;
+ return res.status;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_cmd_register_queue);
+
+u32 t2bce_dma_cmd_unregister_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid)
+{
+ struct bce_queue_cmdq_result_el res;
+ struct bce_cmdq_simple_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res);
+ if (!cmd)
+ return (u32) -1;
+ cmd->cmd = BCE_CMD_UNREGISTER_MEMORY_QUEUE;
+ cmd->flags = 0;
+ cmd->qid = qid;
+ if (t2bce_dma_cmd_finish(cmdq, &res))
+ return (u32)-1;
+ return res.status;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_cmd_unregister_memory_queue);
+
+u32 t2bce_dma_cmd_flush_memory_queue(struct bce_queue_cmdq *cmdq, u16 qid)
+{
+ struct bce_queue_cmdq_result_el res;
+ struct bce_cmdq_simple_memory_queue_cmd *cmd = t2bce_dma_cmd_start(cmdq, &res);
+ if (!cmd)
+ return (u32) -1;
+ cmd->cmd = BCE_CMD_FLUSH_MEMORY_QUEUE;
+ cmd->flags = 0;
+ cmd->qid = qid;
+ if (t2bce_dma_cmd_finish(cmdq, &res))
+ return (u32)-1;
+ return res.status;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_cmd_flush_memory_queue);
+
+static int t2bce_dma_register_queue(struct t2bce_dma_engine *dma, struct bce_queue_memcfg *cfg,
+ const char *name, bool isdirout)
+{
+ if (!dma->ops || !dma->ops->register_queue)
+ return -ENODEV;
+
+ return dma->ops->register_queue(dma->ops_userdata, cfg, name, isdirout);
+}
+
+static int t2bce_dma_unregister_queue(struct t2bce_dma_engine *dma, u16 qid)
+{
+ if (!dma->ops || !dma->ops->unregister_queue)
+ return -ENODEV;
+
+ return dma->ops->unregister_queue(dma->ops_userdata, qid);
+}
+
+static int t2bce_dma_flush_queue(struct t2bce_dma_engine *dma, u16 qid)
+{
+ if (!dma->ops || !dma->ops->flush_queue)
+ return -ENODEV;
+
+ return dma->ops->flush_queue(dma->ops_userdata, qid);
+}
+
+
+struct bce_queue_cq *t2bce_dma_create_cq(struct t2bce_dma_engine *dma, u32 el_count)
+{
+ struct bce_queue_cq *cq;
+ struct bce_queue_memcfg cfg;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0)
+ int qid = ida_simple_get(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL);
+#else
+ int qid = ida_alloc_range(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX - 1, GFP_KERNEL);
+#endif
+ if (qid < 0)
+ return NULL;
+ cq = t2bce_dma_alloc_cq(dma, qid, el_count);
+ if (!cq)
+ return NULL;
+ t2bce_dma_get_cq_memcfg(cq, &cfg);
+ if (t2bce_dma_register_queue(dma, &cfg, NULL, false) != 0) {
+ pr_err("t2bce_dma: CQ registration failed (%i)", qid);
+ t2bce_dma_free_cq(dma, cq);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0)
+ ida_simple_remove(&dma->queue_ida, (uint) qid);
+#else
+ ida_free(&dma->queue_ida, (uint) qid);
+#endif
+ return NULL;
+ }
+ dma->queues[qid] = (struct bce_queue *) cq;
+ return cq;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_create_cq);
+
+struct bce_queue_sq *t2bce_dma_create_sq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name, u32 el_count,
+ int direction, bce_sq_completion compl, void *userdata)
+{
+ struct bce_queue_sq *sq;
+ struct bce_queue_memcfg cfg;
+ int qid;
+ if (cq == NULL)
+ return NULL; /* cq can not be null */
+ if (name == NULL)
+ return NULL; /* name can not be null */
+ if (direction != DMA_TO_DEVICE && direction != DMA_FROM_DEVICE)
+ return NULL; /* unsupported direction */
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0)
+ qid = ida_simple_get(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX, GFP_KERNEL);
+#else
+ qid = ida_alloc_range(&dma->queue_ida, BCE_QUEUE_USER_MIN, BCE_QUEUE_USER_MAX - 1, GFP_KERNEL);
+#endif
+ if (qid < 0)
+ return NULL;
+ sq = t2bce_dma_alloc_sq(dma, qid, sizeof(struct bce_qe_submission), el_count, compl, userdata);
+ if (!sq)
+ return NULL;
+ t2bce_dma_get_sq_memcfg(sq, cq, &cfg);
+ if (t2bce_dma_register_queue(dma, &cfg, name, direction != DMA_FROM_DEVICE) != 0) {
+ pr_err("t2bce_dma: SQ registration failed (%i)", qid);
+ t2bce_dma_free_sq(dma, sq);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0)
+ ida_simple_remove(&dma->queue_ida, (uint) qid);
+#else
+ ida_free(&dma->queue_ida, (uint) qid);
+#endif
+ return NULL;
+ }
+ spin_lock(&dma->queues_lock);
+ dma->queues[qid] = (struct bce_queue *) sq;
+ spin_unlock(&dma->queues_lock);
+ return sq;
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_create_sq);
+
+struct bce_queue_sq *t2bce_dma_create_sq_with_flags(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq, const char *name,
+ u32 el_count, u16 flags, bce_sq_completion compl, void *userdata)
+{
+ int direction = (flags & 1) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+ return t2bce_dma_create_sq(dma, cq, name, el_count, direction, compl, userdata);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_create_sq_with_flags);
+
+void t2bce_dma_destroy_cq(struct t2bce_dma_engine *dma, struct bce_queue_cq *cq)
+{
+ if (!dma->is_being_removed && t2bce_dma_unregister_queue(dma, (u16) cq->qid))
+ pr_err("t2bce_dma: CQ unregister failed");
+ spin_lock(&dma->queues_lock);
+ dma->queues[cq->qid] = NULL;
+ spin_unlock(&dma->queues_lock);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0)
+ ida_simple_remove(&dma->queue_ida, (uint) cq->qid);
+#else
+ ida_free(&dma->queue_ida, (uint) cq->qid);
+#endif
+ t2bce_dma_free_cq(dma, cq);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_destroy_cq);
+
+void t2bce_dma_destroy_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq)
+{
+ if (!dma->is_being_removed && t2bce_dma_unregister_queue(dma, (u16) sq->qid))
+ pr_err("t2bce_dma: SQ unregister failed");
+ spin_lock(&dma->queues_lock);
+ dma->queues[sq->qid] = NULL;
+ spin_unlock(&dma->queues_lock);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,18,0)
+ ida_simple_remove(&dma->queue_ida, (uint) sq->qid);
+#else
+ ida_free(&dma->queue_ida, (uint) sq->qid);
+#endif
+ t2bce_dma_free_sq(dma, sq);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_destroy_sq);
+
+int t2bce_dma_flush_sq(struct t2bce_dma_engine *dma, struct bce_queue_sq *sq)
+{
+ return t2bce_dma_flush_queue(dma, (u16) sq->qid);
+}
+EXPORT_SYMBOL_GPL(t2bce_dma_flush_sq);
+
+static int __init t2bce_dma_module_init(void)
+{
+ pr_info("t2bce_dma: module initialized\n");
+ return 0;
+}
+
+static void __exit t2bce_dma_module_exit(void)
+{
+ pr_info("t2bce_dma: module exited\n");
+}
+
+module_init(t2bce_dma_module_init);
+module_exit(t2bce_dma_module_exit);
+
+MODULE_AUTHOR("André Eikmeyer <andre.eikmeyer@gmail.com>");
+MODULE_DESCRIPTION("Apple T2 BCE DMA queue engine");
+MODULE_VERSION("0.01");
+MODULE_LICENSE("GPL");
diff --git a/drivers/staging/t2bce/t2bce_vhci/Makefile b/drivers/staging/t2bce/t2bce_vhci/Makefile
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_T2BCE_VHCI) += t2bce_vhci.o
+t2bce_vhci-y := vhci.o queue.o transfer.o
+
+ccflags-y += -DWITHOUT_NVME_PATCH
+ccflags-y += -I$(src)
+ccflags-y += -I$(src)/../t2bce_core/include
diff --git a/drivers/staging/t2bce/t2bce_vhci/command.h b/drivers/staging/t2bce/t2bce_vhci/command.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/command.h
@@ -0,0 +1,206 @@
+#ifndef BCE_VHCI_COMMAND_H
+#define BCE_VHCI_COMMAND_H
+
+#include "queue.h"
+#include <linux/jiffies.h>
+#include <linux/usb.h>
+
+#define BCE_VHCI_CMD_TIMEOUT_SHORT msecs_to_jiffies(2000)
+#define BCE_VHCI_CMD_TIMEOUT_LONG msecs_to_jiffies(30000)
+
+#define BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2 2
+#define BCE_VHCI_BULK_MAX_ACTIVE_URBS (1 << BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2)
+
+typedef u8 bce_vhci_port_t;
+typedef u8 bce_vhci_device_t;
+
+enum bce_vhci_command {
+ BCE_VHCI_CMD_CONTROLLER_ENABLE = 1,
+ BCE_VHCI_CMD_CONTROLLER_DISABLE = 2,
+ BCE_VHCI_CMD_CONTROLLER_START = 3,
+ /* Reserved for future controller-level PM work; endpoint PM is used today. */
+ BCE_VHCI_CMD_CONTROLLER_PAUSE = 4,
+
+ BCE_VHCI_CMD_PORT_POWER_ON = 0x10,
+ BCE_VHCI_CMD_PORT_POWER_OFF = 0x11,
+ BCE_VHCI_CMD_PORT_RESUME = 0x12,
+ BCE_VHCI_CMD_PORT_SUSPEND = 0x13,
+ BCE_VHCI_CMD_PORT_RESET = 0x14,
+ BCE_VHCI_CMD_PORT_DISABLE = 0x15,
+ BCE_VHCI_CMD_PORT_STATUS = 0x16,
+ BCE_VHCI_EV_PORT_STATUS_CHANGE = 0x18,
+
+ BCE_VHCI_CMD_DEVICE_CREATE = 0x30,
+ BCE_VHCI_CMD_DEVICE_DESTROY = 0x31,
+
+ BCE_VHCI_CMD_ENDPOINT_CREATE = 0x40,
+ BCE_VHCI_CMD_ENDPOINT_DESTROY = 0x41,
+ BCE_VHCI_CMD_ENDPOINT_SET_STATE = 0x42,
+ BCE_VHCI_CMD_ENDPOINT_RESET = 0x44,
+
+ /* Device to host only */
+ BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE = 0x43,
+ BCE_VHCI_CMD_TRANSFER_REQUEST = 0x1000,
+ BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS = 0x1005
+};
+
+enum bce_vhci_endpoint_state {
+ BCE_VHCI_ENDPOINT_ACTIVE = 0,
+ BCE_VHCI_ENDPOINT_PAUSED = 1,
+ BCE_VHCI_ENDPOINT_STALLED = 2
+};
+
+static inline int bce_vhci_cmd_controller_enable(struct bce_vhci_command_queue *q, u8 busNum, u16 *portMask)
+{
+ int status;
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE;
+ cmd.param1 = 0x7100u | busNum;
+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+ if (!status)
+ *portMask = (u16) res.param2;
+ return status;
+}
+static inline int bce_vhci_cmd_controller_disable(struct bce_vhci_command_queue *q)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+}
+static inline int bce_vhci_cmd_controller_start(struct bce_vhci_command_queue *q)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+}
+static inline int bce_vhci_cmd_controller_pause(struct bce_vhci_command_queue *q)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_CONTROLLER_PAUSE;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+}
+
+static inline int bce_vhci_cmd_port_power_on(struct bce_vhci_command_queue *q, bce_vhci_port_t port)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON;
+ cmd.param1 = port;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+static inline int bce_vhci_cmd_port_power_off(struct bce_vhci_command_queue *q, bce_vhci_port_t port)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_POWER_OFF;
+ cmd.param1 = port;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+static inline int bce_vhci_cmd_port_resume(struct bce_vhci_command_queue *q, bce_vhci_port_t port)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_RESUME;
+ cmd.param1 = port;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+}
+static inline int bce_vhci_cmd_port_suspend(struct bce_vhci_command_queue *q, bce_vhci_port_t port)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_SUSPEND;
+ cmd.param1 = port;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+}
+static inline int bce_vhci_cmd_port_reset(struct bce_vhci_command_queue *q, bce_vhci_port_t port, u32 timeout)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_RESET;
+ cmd.param1 = port;
+ cmd.param2 = timeout;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+static inline int bce_vhci_cmd_port_disable(struct bce_vhci_command_queue *q, bce_vhci_port_t port)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_DISABLE;
+ cmd.param1 = port;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+static inline int bce_vhci_cmd_port_status(struct bce_vhci_command_queue *q, bce_vhci_port_t port,
+ u32 clearFlags, u32 *resStatus)
+{
+ int status;
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_PORT_STATUS;
+ cmd.param1 = port;
+ cmd.param2 = clearFlags & 0x560000;
+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+ if (status >= 0)
+ *resStatus = (u32) res.param2;
+ return status;
+}
+
+static inline int bce_vhci_cmd_device_create(struct bce_vhci_command_queue *q, bce_vhci_port_t port,
+ bce_vhci_device_t *dev)
+{
+ int status;
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE;
+ cmd.param1 = port;
+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+ if (!status)
+ *dev = (bce_vhci_device_t) res.param2;
+ return status;
+}
+static inline int bce_vhci_cmd_device_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY;
+ cmd.param1 = dev;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_LONG);
+}
+
+static inline int bce_vhci_cmd_endpoint_create(struct bce_vhci_command_queue *q, bce_vhci_device_t dev,
+ struct usb_endpoint_descriptor *desc)
+{
+ struct bce_vhci_message cmd, res;
+ int endpoint_type = usb_endpoint_type(desc);
+ int maxp = usb_endpoint_maxp(desc);
+ int maxp_burst = usb_endpoint_maxp_mult(desc) * maxp;
+ u8 max_active_requests_pow2 = 0;
+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE;
+ cmd.param1 = dev | ((desc->bEndpointAddress & 0x8Fu) << 8);
+ if (endpoint_type == USB_ENDPOINT_XFER_BULK)
+ max_active_requests_pow2 = BCE_VHCI_BULK_MAX_ACTIVE_URBS_POW2;
+ cmd.param2 = endpoint_type | ((max_active_requests_pow2 & 0xf) << 4) | (maxp << 16) | ((u64) maxp_burst << 32);
+ if (endpoint_type == USB_ENDPOINT_XFER_INT)
+ cmd.param2 |= (desc->bInterval - 1) << 8;
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+static inline int bce_vhci_cmd_endpoint_destroy(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY;
+ cmd.param1 = dev | (endpoint << 8);
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+static inline int bce_vhci_cmd_endpoint_set_state(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint,
+ enum bce_vhci_endpoint_state newState, enum bce_vhci_endpoint_state *retState)
+{
+ int status;
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_SET_STATE;
+ cmd.param1 = dev | (endpoint << 8);
+ cmd.param2 = (u64) newState;
+ status = bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+ if (status != BCE_VHCI_INTERNAL_ERROR && status != BCE_VHCI_NO_POWER)
+ *retState = (enum bce_vhci_endpoint_state) res.param2;
+ return status;
+}
+static inline int bce_vhci_cmd_endpoint_reset(struct bce_vhci_command_queue *q, bce_vhci_device_t dev, u8 endpoint)
+{
+ struct bce_vhci_message cmd, res;
+ cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET;
+ cmd.param1 = dev | (endpoint << 8);
+ return bce_vhci_command_queue_execute(q, &cmd, &res, BCE_VHCI_CMD_TIMEOUT_SHORT);
+}
+
+
+#endif //BCE_VHCI_COMMAND_H
diff --git a/drivers/staging/t2bce/t2bce_vhci/queue.c b/drivers/staging/t2bce/t2bce_vhci/queue.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/queue.c
@@ -0,0 +1,269 @@
+#include "queue.h"
+#include "vhci.h"
+
+#include <linux/dma-mapping.h>
+
+
+static void bce_vhci_message_queue_completion(struct t2bce_core_queue_sq *sq);
+
+int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name)
+{
+ int status;
+ ret->cq = t2bce_core_create_cq(vhci->client, VHCI_EVENT_QUEUE_EL_COUNT);
+ if (!ret->cq)
+ return -EINVAL;
+ ret->sq = t2bce_core_create_sq(vhci->client, ret->cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_TO_DEVICE,
+ bce_vhci_message_queue_completion, ret);
+ if (!ret->sq) {
+ status = -EINVAL;
+ goto fail_cq;
+ }
+ ret->data = dma_alloc_coherent(t2bce_core_client_dma_dev(vhci->client),
+ sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT,
+ &ret->dma_addr, GFP_KERNEL);
+ if (!ret->data) {
+ status = -EINVAL;
+ goto fail_sq;
+ }
+ return 0;
+
+fail_sq:
+ t2bce_core_destroy_sq(vhci->client, ret->sq);
+ ret->sq = NULL;
+fail_cq:
+ t2bce_core_destroy_cq(vhci->client, ret->cq);
+ ret->cq = NULL;
+ return status;
+}
+
+void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q)
+{
+ if (!q->cq)
+ return;
+ dma_free_coherent(t2bce_core_client_dma_dev(vhci->client),
+ sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT,
+ q->data, q->dma_addr);
+ t2bce_core_destroy_sq(vhci->client, q->sq);
+ t2bce_core_destroy_cq(vhci->client, q->cq);
+}
+
+void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req)
+{
+ int sidx;
+ sidx = t2bce_core_queue_sq_tail(q->sq);
+ pr_debug("t2bce_vhci: Send message: %x s=%x p1=%x p2=%llx\n", req->cmd, req->status, req->param1, req->param2);
+ q->data[sidx] = *req;
+ t2bce_core_set_next_submission_single(q->sq, q->dma_addr + sizeof(struct bce_vhci_message) * sidx,
+ sizeof(struct bce_vhci_message));
+ t2bce_core_submit_to_device(q->sq);
+}
+
+static void bce_vhci_message_queue_completion(struct t2bce_core_queue_sq *sq)
+{
+ while (t2bce_core_next_completion(sq))
+ t2bce_core_notify_submission_complete(sq);
+}
+
+
+
+static void bce_vhci_event_queue_completion(struct t2bce_core_queue_sq *sq);
+
+int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name,
+ t2bce_core_sq_completion compl)
+{
+ ret->vhci = vhci;
+
+ ret->sq = t2bce_core_create_sq(vhci->client, vhci->ev_cq, name, VHCI_EVENT_QUEUE_EL_COUNT, DMA_FROM_DEVICE, compl, ret);
+ if (!ret->sq)
+ return -EINVAL;
+ ret->data = dma_alloc_coherent(t2bce_core_client_dma_dev(vhci->client),
+ sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT,
+ &ret->dma_addr, GFP_KERNEL);
+ if (!ret->data) {
+ t2bce_core_destroy_sq(vhci->client, ret->sq);
+ ret->sq = NULL;
+ return -EINVAL;
+ }
+
+ init_completion(&ret->queue_empty_completion);
+ bce_vhci_event_queue_submit_pending(ret, VHCI_EVENT_PENDING_COUNT);
+ return 0;
+}
+
+int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name,
+ bce_vhci_event_queue_callback cb)
+{
+ ret->cb = cb;
+ return __bce_vhci_event_queue_create(vhci, ret, name, bce_vhci_event_queue_completion);
+}
+
+void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q)
+{
+ if (!q->sq)
+ return;
+ dma_free_coherent(t2bce_core_client_dma_dev(vhci->client),
+ sizeof(struct bce_vhci_message) * VHCI_EVENT_QUEUE_EL_COUNT,
+ q->data, q->dma_addr);
+ t2bce_core_destroy_sq(vhci->client, q->sq);
+}
+
+static void bce_vhci_event_queue_completion(struct t2bce_core_queue_sq *sq)
+{
+ struct t2bce_core_sq_completion_data *cd;
+ struct bce_vhci_event_queue *ev = t2bce_core_queue_sq_userdata(sq);
+ struct bce_vhci_message *msg;
+ size_t cnt = 0;
+
+ while ((cd = t2bce_core_next_completion(sq))) {
+ if (cd->status == T2BCE_COMPLETION_ABORTED) {
+ t2bce_core_notify_submission_complete(sq);
+ continue;
+ }
+ msg = &ev->data[t2bce_core_queue_sq_head(sq)];
+ pr_debug("t2bce_vhci: Got event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2);
+ ev->cb(ev, msg);
+
+ t2bce_core_notify_submission_complete(sq);
+ ++cnt;
+ }
+ bce_vhci_event_queue_submit_pending(ev, cnt);
+ if (t2bce_core_queue_sq_available(sq) == t2bce_core_queue_sq_capacity(sq) - 1)
+ complete(&ev->queue_empty_completion);
+}
+
+void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count)
+{
+ int idx;
+ while (count--) {
+ if (t2bce_core_reserve_submission(q->sq, NULL)) {
+ pr_err("t2bce_vhci: Failed to reserve an event queue submission\n");
+ break;
+ }
+ idx = t2bce_core_queue_sq_tail(q->sq);
+ t2bce_core_set_next_submission_single(q->sq,
+ q->dma_addr + idx * sizeof(struct bce_vhci_message), sizeof(struct bce_vhci_message));
+ }
+ t2bce_core_submit_to_device(q->sq);
+}
+
+void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q)
+{
+ unsigned long timeout;
+ reinit_completion(&q->queue_empty_completion);
+ if (t2bce_core_flush_queue(q->vhci->client, q->sq))
+ pr_warn("t2bce_vhci: failed to flush event queue\n");
+ timeout = msecs_to_jiffies(5000);
+ while (t2bce_core_queue_sq_available(q->sq) != t2bce_core_queue_sq_capacity(q->sq) - 1) {
+ timeout = wait_for_completion_timeout(&q->queue_empty_completion, timeout);
+ if (timeout == 0) {
+ pr_err("t2bce_vhci: waiting for queue to be flushed timed out\n");
+ break;
+ }
+ }
+}
+
+void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q)
+{
+ if (t2bce_core_queue_sq_available(q->sq) != t2bce_core_queue_sq_capacity(q->sq) - 1) {
+ pr_err("t2bce_vhci: resume of a queue with pending submissions\n");
+ return;
+ }
+ bce_vhci_event_queue_submit_pending(q, VHCI_EVENT_PENDING_COUNT);
+}
+
+void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq)
+{
+ ret->mq = mq;
+ ret->completion.result = NULL;
+ init_completion(&ret->completion.completion);
+ spin_lock_init(&ret->completion_lock);
+ mutex_init(&ret->mutex);
+}
+
+void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq)
+{
+ spin_lock(&cq->completion_lock);
+ if (cq->completion.result) {
+ memset(cq->completion.result, 0, sizeof(struct bce_vhci_message));
+ cq->completion.result->status = BCE_VHCI_ABORT;
+ complete(&cq->completion.completion);
+ cq->completion.result = NULL;
+ }
+ spin_unlock(&cq->completion_lock);
+ mutex_lock(&cq->mutex);
+ mutex_unlock(&cq->mutex);
+ mutex_destroy(&cq->mutex);
+}
+
+void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg)
+{
+ struct bce_vhci_command_queue_completion *c = &cq->completion;
+
+ spin_lock(&cq->completion_lock);
+ if (c->result) {
+ *c->result = *msg;
+ complete(&c->completion);
+ c->result = NULL;
+ }
+ spin_unlock(&cq->completion_lock);
+}
+
+static int __bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req,
+ struct bce_vhci_message *res, unsigned long timeout)
+{
+ int status;
+ struct bce_vhci_command_queue_completion *c;
+ struct bce_vhci_message creq;
+ c = &cq->completion;
+
+ if ((status = t2bce_core_reserve_submission(cq->mq->sq, &timeout)))
+ return status;
+
+ spin_lock(&cq->completion_lock);
+ c->result = res;
+ reinit_completion(&c->completion);
+ spin_unlock(&cq->completion_lock);
+
+ bce_vhci_message_queue_write(cq->mq, req);
+
+ if (!wait_for_completion_timeout(&c->completion, timeout)) {
+ /* Ask bridgeOS to cancel the command before reporting timeout. */
+ pr_debug("t2bce_vhci: command timed out req=%x\n", req->cmd);
+ if ((status = t2bce_core_reserve_submission(cq->mq->sq, &timeout)))
+ return status;
+
+ creq = *req;
+ creq.cmd |= 0x4000;
+ bce_vhci_message_queue_write(cq->mq, &creq);
+
+ if (!wait_for_completion_timeout(&c->completion, 1000)) {
+ pr_err("t2bce_vhci: Possible desync, cmd cancel timed out\n");
+
+ spin_lock(&cq->completion_lock);
+ c->result = NULL;
+ spin_unlock(&cq->completion_lock);
+ return -ETIMEDOUT;
+ }
+ if ((res->cmd & ~0x8000) == creq.cmd)
+ return -ETIMEDOUT;
+ /* The original reply arrived after the cancellation was queued. */
+ }
+
+ if ((res->cmd & ~0x8000) != req->cmd) {
+ pr_err("t2bce_vhci: Possible desync, cmd reply mismatch req=%x, res=%x\n", req->cmd, res->cmd);
+ return -EIO;
+ }
+ if (res->status == BCE_VHCI_SUCCESS)
+ return 0;
+ return res->status;
+}
+
+int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req,
+ struct bce_vhci_message *res, unsigned long timeout)
+{
+ int status;
+ mutex_lock(&cq->mutex);
+ status = __bce_vhci_command_queue_execute(cq, req, res, timeout);
+ mutex_unlock(&cq->mutex);
+ return status;
+}
diff --git a/drivers/staging/t2bce/t2bce_vhci/queue.h b/drivers/staging/t2bce/t2bce_vhci/queue.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/queue.h
@@ -0,0 +1,76 @@
+#ifndef BCE_VHCI_QUEUE_H
+#define BCE_VHCI_QUEUE_H
+
+#include <linux/completion.h>
+#include "t2bce_core_transport.h"
+
+#define VHCI_EVENT_QUEUE_EL_COUNT 256
+#define VHCI_EVENT_PENDING_COUNT 32
+
+struct bce_vhci;
+struct bce_vhci_event_queue;
+
+enum bce_vhci_message_status {
+ BCE_VHCI_SUCCESS = 1,
+ BCE_VHCI_ERROR = 2,
+ BCE_VHCI_USB_PIPE_STALL = 3,
+ BCE_VHCI_ABORT = 4,
+ BCE_VHCI_BAD_ARGUMENT = 5,
+ BCE_VHCI_OVERRUN = 6,
+ BCE_VHCI_INTERNAL_ERROR = 7,
+ BCE_VHCI_NO_POWER = 8,
+ BCE_VHCI_UNSUPPORTED = 9
+};
+struct bce_vhci_message {
+ u16 cmd;
+ u16 status; // bce_vhci_message_status
+ u32 param1;
+ u64 param2;
+};
+
+struct bce_vhci_message_queue {
+ struct t2bce_core_queue_cq *cq;
+ struct t2bce_core_queue_sq *sq;
+ struct bce_vhci_message *data;
+ dma_addr_t dma_addr;
+};
+typedef void (*bce_vhci_event_queue_callback)(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg);
+struct bce_vhci_event_queue {
+ struct bce_vhci *vhci;
+ struct t2bce_core_queue_sq *sq;
+ struct bce_vhci_message *data;
+ dma_addr_t dma_addr;
+ bce_vhci_event_queue_callback cb;
+ struct completion queue_empty_completion;
+};
+struct bce_vhci_command_queue_completion {
+ struct bce_vhci_message *result;
+ struct completion completion;
+};
+struct bce_vhci_command_queue {
+ struct bce_vhci_message_queue *mq;
+ struct bce_vhci_command_queue_completion completion;
+ struct spinlock completion_lock;
+ struct mutex mutex;
+};
+
+int bce_vhci_message_queue_create(struct bce_vhci *vhci, struct bce_vhci_message_queue *ret, const char *name);
+void bce_vhci_message_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_message_queue *q);
+void bce_vhci_message_queue_write(struct bce_vhci_message_queue *q, struct bce_vhci_message *req);
+
+int __bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name,
+ t2bce_core_sq_completion compl);
+int bce_vhci_event_queue_create(struct bce_vhci *vhci, struct bce_vhci_event_queue *ret, const char *name,
+ bce_vhci_event_queue_callback cb);
+void bce_vhci_event_queue_destroy(struct bce_vhci *vhci, struct bce_vhci_event_queue *q);
+void bce_vhci_event_queue_submit_pending(struct bce_vhci_event_queue *q, size_t count);
+void bce_vhci_event_queue_pause(struct bce_vhci_event_queue *q);
+void bce_vhci_event_queue_resume(struct bce_vhci_event_queue *q);
+
+void bce_vhci_command_queue_create(struct bce_vhci_command_queue *ret, struct bce_vhci_message_queue *mq);
+void bce_vhci_command_queue_destroy(struct bce_vhci_command_queue *cq);
+int bce_vhci_command_queue_execute(struct bce_vhci_command_queue *cq, struct bce_vhci_message *req,
+ struct bce_vhci_message *res, unsigned long timeout);
+void bce_vhci_command_queue_deliver_completion(struct bce_vhci_command_queue *cq, struct bce_vhci_message *msg);
+
+#endif //BCE_VHCI_QUEUE_H
diff --git a/drivers/staging/t2bce/t2bce_vhci/transfer.c b/drivers/staging/t2bce/t2bce_vhci/transfer.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/transfer.c
@@ -0,0 +1,1261 @@
+#include "transfer.h"
+#include "vhci.h"
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include <linux/usb/hcd.h>
+
+static void bce_vhci_transfer_queue_completion(struct t2bce_core_queue_sq *sq);
+static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q);
+static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q);
+
+static int bce_vhci_urb_init(struct bce_vhci_urb *vurb);
+static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg);
+static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c);
+static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status);
+static void bce_vhci_urb_free(struct bce_vhci_urb *urb);
+
+static void bce_vhci_transfer_queue_reset_w(struct work_struct *work);
+static void bce_vhci_transfer_queue_resume_w(struct work_struct *work);
+static void bce_vhci_transfer_queue_cancel_w(struct work_struct *work);
+
+void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q,
+ struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir)
+{
+ char name[0x21];
+ INIT_LIST_HEAD(&q->evq);
+ INIT_LIST_HEAD(&q->giveback_urb_list);
+ INIT_LIST_HEAD(&q->cancel_urb_list);
+ spin_lock_init(&q->urb_lock);
+ mutex_init(&q->pause_lock);
+ init_waitqueue_head(&q->sq_out_wait_queue);
+ atomic_set(&q->sq_out_pending, 0);
+ q->vhci = vhci;
+ q->endp = endp;
+ q->dev_addr = dev_addr;
+ q->endp_addr = (u8) (endp->desc.bEndpointAddress & 0x8F);
+ q->state = BCE_VHCI_ENDPOINT_ACTIVE;
+ q->active = true;
+ q->stalled = false;
+ q->paused_by = 0;
+ q->max_active_requests = 1;
+ if (usb_endpoint_type(&endp->desc) == USB_ENDPOINT_XFER_BULK)
+ q->max_active_requests = BCE_VHCI_BULK_MAX_ACTIVE_URBS;
+ q->remaining_active_requests = q->max_active_requests;
+ q->cq = t2bce_core_create_cq(vhci->client, 0x100);
+ INIT_WORK(&q->w_reset, bce_vhci_transfer_queue_reset_w);
+ INIT_WORK(&q->w_resume, bce_vhci_transfer_queue_resume_w);
+ INIT_WORK(&q->w_cancel, bce_vhci_transfer_queue_cancel_w);
+ q->sq_in = NULL;
+ if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
+ snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, 0x80 | usb_endpoint_num(&endp->desc));
+ q->sq_in = t2bce_core_create_sq(vhci->client, q->cq, name, 0x100, DMA_FROM_DEVICE,
+ bce_vhci_transfer_queue_completion, q);
+ }
+ q->sq_out = NULL;
+ if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
+ snprintf(name, sizeof(name), "VHC1-%i-%02x", dev_addr, usb_endpoint_num(&endp->desc));
+ q->sq_out = t2bce_core_create_sq(vhci->client, q->cq, name, 0x100, DMA_TO_DEVICE,
+ bce_vhci_transfer_queue_completion, q);
+ }
+}
+
+void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q)
+{
+ cancel_work_sync(&q->w_resume);
+ cancel_work_sync(&q->w_reset);
+ cancel_work_sync(&q->w_cancel);
+ bce_vhci_transfer_queue_giveback(q);
+ bce_vhci_transfer_queue_remove_pending(q);
+ if (q->sq_in)
+ t2bce_core_destroy_sq(vhci->client, q->sq_in);
+ if (q->sq_out)
+ t2bce_core_destroy_sq(vhci->client, q->sq_out);
+ t2bce_core_destroy_cq(vhci->client, q->cq);
+}
+
+static inline bool bce_vhci_transfer_queue_can_init_urb(struct bce_vhci_transfer_queue *q)
+{
+ return q->remaining_active_requests > 0;
+}
+
+static bool bce_vhci_transfer_queue_defer_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg)
+{
+ struct bce_vhci_list_message *lm;
+ lm = kmalloc(sizeof(struct bce_vhci_list_message), GFP_ATOMIC);
+ if (!lm) {
+ pr_warn("t2bce_vhci: [%02x] failed to defer transfer queue event cmd=%x status=%x\n",
+ q->endp_addr, msg->cmd, msg->status);
+ return false;
+ }
+ INIT_LIST_HEAD(&lm->list);
+ lm->msg = *msg;
+ list_add_tail(&lm->list, &q->evq);
+ return true;
+}
+
+static void bce_vhci_transfer_queue_giveback(struct bce_vhci_transfer_queue *q)
+{
+ unsigned long flags;
+ struct urb *urb;
+ spin_lock_irqsave(&q->urb_lock, flags);
+ while (!list_empty(&q->giveback_urb_list)) {
+ urb = list_first_entry(&q->giveback_urb_list, struct urb, urb_list);
+ list_del(&urb->urb_list);
+
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ usb_hcd_giveback_urb(q->vhci->hcd, urb, urb->status);
+ spin_lock_irqsave(&q->urb_lock, flags);
+ }
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+}
+
+static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q);
+
+static int bce_vhci_transfer_queue_port(struct bce_vhci_transfer_queue *q)
+{
+ int port;
+
+ for (port = 1; port <= q->vhci->port_count &&
+ port < ARRAY_SIZE(q->vhci->port_to_device); port++) {
+ if (q->vhci->port_to_device[port] == q->dev_addr)
+ return port;
+ }
+
+ return 0;
+}
+
+static bool bce_vhci_transfer_queue_is_ep0(struct bce_vhci_transfer_queue *q)
+{
+ return q->endp_addr == 0x00;
+}
+
+static size_t bce_vhci_transfer_queue_event_count(struct bce_vhci_transfer_queue *q)
+{
+ struct bce_vhci_list_message *lm;
+ size_t count = 0;
+
+ list_for_each_entry(lm, &q->evq, list)
+ count++;
+
+ return count;
+}
+
+static void bce_vhci_transfer_queue_log_event(struct bce_vhci_transfer_queue *q,
+ struct bce_vhci_message *msg, const char *stage)
+{
+ if (!bce_vhci_transfer_queue_is_ep0(q))
+ return;
+
+ pr_debug("t2bce_vhci: EP0 event %s dev=%u port=%d cmd=%x msg_status=%x p1=%x p2=%llx active=%u paused_by=%x stalled=%u qstate=%u remaining=%u evq=%zu urb_empty=%d\n",
+ stage, q->dev_addr, bce_vhci_transfer_queue_port(q),
+ msg->cmd, msg->status, msg->param1, msg->param2,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests,
+ bce_vhci_transfer_queue_event_count(q),
+ list_empty(&q->endp->urb_list));
+}
+
+static void bce_vhci_transfer_queue_log_completion(struct bce_vhci_transfer_queue *q,
+ struct t2bce_core_queue_sq *sq, struct t2bce_core_sq_completion_data *c, const char *stage)
+{
+ const char *sq_name = "unknown";
+
+ if (!bce_vhci_transfer_queue_is_ep0(q))
+ return;
+
+ if (sq == q->sq_in)
+ sq_name = "in";
+ else if (sq == q->sq_out)
+ sq_name = "out";
+
+ pr_debug("t2bce_vhci: EP0 completion %s dev=%u port=%d sq=%s cstatus=%x data_size=%llu sq_out_pending=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u evq=%zu urb_empty=%d\n",
+ stage, q->dev_addr, bce_vhci_transfer_queue_port(q),
+ sq_name, c->status, c->data_size,
+ atomic_read(&q->sq_out_pending), q->active, q->paused_by,
+ q->stalled, q->state, q->remaining_active_requests,
+ bce_vhci_transfer_queue_event_count(q),
+ list_empty(&q->endp->urb_list));
+}
+
+static void bce_vhci_urb_log_control(struct bce_vhci_urb *urb, const char *stage)
+{
+ struct bce_vhci_transfer_queue *q = urb->q;
+ struct usb_ctrlrequest *ctrl;
+
+ if (!urb->is_control || !bce_vhci_transfer_queue_is_ep0(q))
+ return;
+
+ ctrl = (struct usb_ctrlrequest *) urb->urb->setup_packet;
+ if (!ctrl) {
+ pr_debug("t2bce_vhci: EP0 control %s dev=%u port=%d urb_state=%u no-setup xfer_len=%u actual=%u send=%u recv=%u status=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u urb_reject=%d\n",
+ stage, q->dev_addr, bce_vhci_transfer_queue_port(q),
+ urb->state, urb->urb->transfer_buffer_length,
+ urb->urb->actual_length, urb->send_offset,
+ urb->receive_offset, urb->received_status, q->active,
+ q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests, atomic_read(&urb->urb->reject));
+ return;
+ }
+
+ /* Per-stage EP0 control-transfer trace.
+ * Keep this at pr_debug; enable dynamic debug when diagnosing
+ * setup/data/status ordering or endpoint recovery issues.
+ */
+ pr_debug("t2bce_vhci: EP0 control %s dev=%u port=%d urb_state=%u dir=%s reqtype=%02x req=%02x value=%04x index=%04x wlen=%u xfer_len=%u actual=%u send=%u recv=%u status=%d active=%u paused_by=%x stalled=%u qstate=%u remaining=%u urb_reject=%d\n",
+ stage, q->dev_addr, bce_vhci_transfer_queue_port(q),
+ urb->state, usb_urb_dir_in(urb->urb) ? "in" : "out",
+ ctrl->bRequestType, ctrl->bRequest,
+ le16_to_cpu(ctrl->wValue), le16_to_cpu(ctrl->wIndex),
+ le16_to_cpu(ctrl->wLength), urb->urb->transfer_buffer_length,
+ urb->urb->actual_length, urb->send_offset, urb->receive_offset,
+ urb->received_status, q->active, q->paused_by, q->stalled,
+ q->state, q->remaining_active_requests,
+ atomic_read(&urb->urb->reject));
+}
+
+static void bce_vhci_transfer_queue_deliver_pending(struct bce_vhci_transfer_queue *q)
+{
+ struct urb *urb;
+ struct bce_vhci_list_message *lm;
+
+ while (!list_empty(&q->endp->urb_list) && !list_empty(&q->evq)) {
+ urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list);
+
+ lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list);
+ bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-deliver");
+ if (bce_vhci_urb_update(urb->hcpriv, &lm->msg) == -EAGAIN) {
+ bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-still-waiting");
+ break;
+ }
+ bce_vhci_transfer_queue_log_event(q, &lm->msg, "deferred-consumed");
+ list_del(&lm->list);
+ kfree(lm);
+ }
+
+ /* some of the URBs could have been completed, so initialize more URBs if possible */
+ bce_vhci_transfer_queue_init_pending_urbs(q);
+}
+
+static void bce_vhci_transfer_queue_remove_pending(struct bce_vhci_transfer_queue *q)
+{
+ unsigned long flags;
+ struct bce_vhci_list_message *lm;
+ spin_lock_irqsave(&q->urb_lock, flags);
+ if (bce_vhci_transfer_queue_is_ep0(q) && !list_empty(&q->evq))
+ pr_debug("t2bce_vhci: EP0 event remove-pending dev=%u port=%d evq=%zu active=%u paused_by=%x stalled=%u qstate=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q),
+ bce_vhci_transfer_queue_event_count(q), q->active,
+ q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+ while (!list_empty(&q->evq)) {
+ lm = list_first_entry(&q->evq, struct bce_vhci_list_message, list);
+ list_del(&lm->list);
+ kfree(lm);
+ }
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+}
+
+void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg)
+{
+ unsigned long flags;
+ struct bce_vhci_urb *turb;
+ struct urb *urb;
+ spin_lock_irqsave(&q->urb_lock, flags);
+ bce_vhci_transfer_queue_log_event(q, msg, "incoming");
+ /* Paused queues may still complete in-flight work but must not deliver new work. */
+ if (!q->active) {
+ if (bce_vhci_transfer_queue_is_ep0(q) &&
+ msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST &&
+ (q->paused_by & BCE_VHCI_PAUSE_INTERNAL_WQ)) {
+ bce_vhci_transfer_queue_log_event(q, msg, "defer-inactive-internal-pause");
+ if (!bce_vhci_transfer_queue_defer_event(q, msg))
+ bce_vhci_transfer_queue_log_event(q, msg, "defer-inactive-failed");
+ } else {
+ bce_vhci_transfer_queue_log_event(q, msg, "drop-inactive");
+ }
+ goto complete;
+ }
+ bce_vhci_transfer_queue_deliver_pending(q);
+
+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST &&
+ (!list_empty(&q->evq) || list_empty(&q->endp->urb_list))) {
+ bce_vhci_transfer_queue_log_event(q, msg, "defer-no-urb-or-backlog");
+ if (!bce_vhci_transfer_queue_defer_event(q, msg))
+ bce_vhci_transfer_queue_log_event(q, msg, "defer-no-urb-or-backlog-failed");
+ goto complete;
+ }
+ if (list_empty(&q->endp->urb_list)) {
+ bce_vhci_transfer_queue_log_event(q, msg, "unexpected-empty");
+ pr_err("t2bce_vhci: [%02x] Unexpected transfer queue event\n", q->endp_addr);
+ goto complete;
+ }
+ urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list);
+ turb = urb->hcpriv;
+ if (bce_vhci_urb_update(turb, msg) == -EAGAIN) {
+ bce_vhci_transfer_queue_log_event(q, msg, "defer-eagain");
+ if (!bce_vhci_transfer_queue_defer_event(q, msg))
+ bce_vhci_transfer_queue_log_event(q, msg, "defer-eagain-failed");
+ } else {
+ bce_vhci_transfer_queue_log_event(q, msg, "consumed");
+ bce_vhci_transfer_queue_init_pending_urbs(q);
+ }
+
+complete:
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ bce_vhci_transfer_queue_giveback(q);
+}
+
+static void bce_vhci_transfer_queue_completion(struct t2bce_core_queue_sq *sq)
+{
+ unsigned long flags;
+ struct t2bce_core_sq_completion_data *c;
+ struct urb *urb;
+ struct bce_vhci_transfer_queue *q = t2bce_core_queue_sq_userdata(sq);
+ bool is_sq_out = (sq == q->sq_out);
+ spin_lock_irqsave(&q->urb_lock, flags);
+ while ((c = t2bce_core_next_completion(sq))) {
+ if (c->status == T2BCE_COMPLETION_ABORTED) { /* We flushed the queue */
+ bce_vhci_transfer_queue_log_completion(q, sq, c, "aborted");
+ pr_debug("t2bce_vhci: [%02x] Got an abort completion\n", q->endp_addr);
+ if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0)
+ wake_up(&q->sq_out_wait_queue);
+ t2bce_core_notify_submission_complete(sq);
+ continue;
+ }
+ if (list_empty(&q->endp->urb_list)) {
+ bce_vhci_transfer_queue_log_completion(q, sq, c, "empty");
+ pr_err("t2bce_vhci: [%02x] Got a completion while no requests are pending\n", q->endp_addr);
+ if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0)
+ wake_up(&q->sq_out_wait_queue);
+ continue;
+ }
+ bce_vhci_transfer_queue_log_completion(q, sq, c, "incoming");
+ pr_debug("t2bce_vhci: [%02x] Got a transfer queue completion\n", q->endp_addr);
+ urb = list_first_entry(&q->endp->urb_list, struct urb, urb_list);
+ bce_vhci_urb_transfer_completion(urb->hcpriv, c);
+ if (is_sq_out && atomic_dec_if_positive(&q->sq_out_pending) == 0)
+ wake_up(&q->sq_out_wait_queue);
+ t2bce_core_notify_submission_complete(sq);
+ }
+ bce_vhci_transfer_queue_deliver_pending(q);
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ bce_vhci_transfer_queue_giveback(q);
+}
+
+int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q)
+{
+ unsigned long flags;
+ int status;
+ u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F);
+ int pending;
+ long timeout;
+ spin_lock_irqsave(&q->urb_lock, flags);
+ q->active = false;
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ if (q->sq_out) {
+ /* Let pending OUT submissions drain before the endpoint is paused. */
+ pending = atomic_read(&q->sq_out_pending);
+ if (pending > 0) {
+ timeout = wait_event_timeout(q->sq_out_wait_queue,
+ atomic_read(&q->sq_out_pending) == 0,
+ msecs_to_jiffies(2000));
+ if (!timeout && atomic_read(&q->sq_out_pending) > 0)
+ pr_warn("t2bce_vhci: [%02x] pause timeout waiting for %d outputs\n",
+ q->endp_addr, atomic_read(&q->sq_out_pending));
+ }
+ }
+ bce_vhci_transfer_queue_remove_pending(q);
+ if ((status = bce_vhci_cmd_endpoint_set_state(
+ &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state)))
+ return status;
+ if (q->state != BCE_VHCI_ENDPOINT_PAUSED)
+ return -EINVAL;
+ if (q->sq_in)
+ t2bce_core_flush_queue(q->vhci->client, q->sq_in);
+ if (q->sq_out)
+ t2bce_core_flush_queue(q->vhci->client, q->sq_out);
+ return 0;
+}
+
+static void bce_vhci_urb_resume(struct bce_vhci_urb *urb);
+
+int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q)
+{
+ unsigned long flags;
+ int status;
+ struct urb *urb, *urbt;
+ struct bce_vhci_urb *vurb;
+ u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F);
+ if (bce_vhci_transfer_queue_is_ep0(q))
+ pr_debug("t2bce_vhci: tq resume begin dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+ if ((status = bce_vhci_cmd_endpoint_set_state(
+ &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_ACTIVE, &q->state))) {
+ pr_warn("t2bce_vhci: tq resume set-active failed dev=%u port=%d ep=%02x status=%d ret_state=%u active=%u paused_by=%x stalled=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ status, q->state, q->active, q->paused_by, q->stalled);
+ return status;
+ }
+ if (q->state != BCE_VHCI_ENDPOINT_ACTIVE) {
+ pr_warn("t2bce_vhci: tq resume set-active returned non-active dev=%u port=%d ep=%02x ret_state=%u active=%u paused_by=%x stalled=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->state, q->active, q->paused_by, q->stalled);
+ return -EINVAL;
+ }
+ spin_lock_irqsave(&q->urb_lock, flags);
+ q->active = true;
+ list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) {
+ vurb = urb->hcpriv;
+ if (vurb->state == BCE_VHCI_URB_INIT_PENDING) {
+ if (!bce_vhci_transfer_queue_can_init_urb(q))
+ break;
+ bce_vhci_urb_init(vurb);
+ } else {
+ bce_vhci_urb_resume(vurb);
+ }
+ }
+ bce_vhci_transfer_queue_deliver_pending(q);
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ if (bce_vhci_transfer_queue_is_ep0(q))
+ pr_debug("t2bce_vhci: tq resume done dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+ return 0;
+}
+
+int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src)
+{
+ int ret = 0;
+ u32 old_paused_by;
+
+ mutex_lock(&q->pause_lock);
+ old_paused_by = q->paused_by;
+ if ((q->paused_by & src) != src) {
+ if (!q->paused_by)
+ ret = bce_vhci_transfer_queue_do_pause(q);
+ if (!ret)
+ q->paused_by |= src;
+ }
+ if (bce_vhci_transfer_queue_is_ep0(q) || ret)
+ pr_debug("t2bce_vhci: tq pause dev=%u port=%d ep=%02x src=%x ret=%d paused_by=%x->%x active=%u stalled=%u state=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ src, ret, old_paused_by, q->paused_by, q->active,
+ q->stalled, q->state);
+ mutex_unlock(&q->pause_lock);
+ return ret;
+}
+
+int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src)
+{
+ int ret = 0;
+ u32 old_paused_by;
+ bool did_resume = false;
+ bool cleared_shutdown = false;
+
+ mutex_lock(&q->pause_lock);
+ old_paused_by = q->paused_by;
+ /*
+ * EP0 queues can carry the old shutdown pause across preserved-state
+ * resume. System resume owns that transition, so clear the shutdown owner
+ * here before deciding whether the queue needs a firmware resume.
+ */
+ if (src == BCE_VHCI_PAUSE_SUSPEND &&
+ q->endp_addr == 0x00 &&
+ q->paused_by == (BCE_VHCI_PAUSE_SHUTDOWN | BCE_VHCI_PAUSE_SUSPEND)) {
+ q->paused_by &= ~BCE_VHCI_PAUSE_SHUTDOWN;
+ cleared_shutdown = true;
+ }
+ if (q->paused_by & src) {
+ if (!(q->paused_by & ~src)) {
+ ret = bce_vhci_transfer_queue_do_resume(q);
+ did_resume = true;
+ }
+ if (!ret)
+ q->paused_by &= ~src;
+ }
+ if (src == BCE_VHCI_PAUSE_SUSPEND)
+ pr_debug("t2bce_vhci: system resume dev=%u ep=%02x src=%x paused_by=%x->%x firmware_resume=%u cleared_shutdown=%u ret=%d state=%x active=%u\n",
+ q->dev_addr, q->endp_addr, src, old_paused_by, q->paused_by,
+ did_resume, cleared_shutdown, ret, q->state, q->active);
+ else if (q->endp_addr == 0x00)
+ pr_debug("t2bce_vhci: EP0 resume dev=%u src=%x paused_by=%x->%x did_resume=%u ret=%d state=%x active=%u\n",
+ q->dev_addr, src, old_paused_by, q->paused_by, did_resume,
+ ret, q->state, q->active);
+ mutex_unlock(&q->pause_lock);
+ return ret;
+}
+
+int bce_vhci_transfer_queue_suspend_pause(struct bce_vhci_transfer_queue *q)
+{
+ unsigned long flags;
+ int status;
+ u8 endp_addr = (u8) (q->endp->desc.bEndpointAddress & 0x8F);
+ int pending;
+ long timeout;
+ int ret = 0;
+ u32 old_paused_by;
+
+ mutex_lock(&q->pause_lock);
+ old_paused_by = q->paused_by;
+ if ((q->paused_by & BCE_VHCI_PAUSE_SUSPEND) == BCE_VHCI_PAUSE_SUSPEND) {
+ pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=already-suspend paused_by=%x state=%x active=%u\n",
+ q->dev_addr, q->endp_addr, q->paused_by, q->state, q->active);
+ goto out;
+ }
+ if (q->paused_by) {
+ /*
+ * Another owner already paused the endpoint. In that case system
+ * suspend only records its own ownership bit and must not send a
+ * second firmware pause for the same queue.
+ */
+ q->paused_by |= BCE_VHCI_PAUSE_SUSPEND;
+ pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=mark-only paused_by=%x->%x state=%x active=%u\n",
+ q->dev_addr, q->endp_addr, old_paused_by, q->paused_by,
+ q->state, q->active);
+ goto out;
+ }
+
+ spin_lock_irqsave(&q->urb_lock, flags);
+ q->active = false;
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ if (q->sq_out) {
+ pending = atomic_read(&q->sq_out_pending);
+ if (pending > 0) {
+ timeout = wait_event_timeout(q->sq_out_wait_queue,
+ atomic_read(&q->sq_out_pending) == 0,
+ msecs_to_jiffies(2000));
+ if (!timeout && atomic_read(&q->sq_out_pending) > 0)
+ pr_warn("t2bce_vhci: [%02x] pause timeout waiting for %d outputs\n",
+ q->endp_addr, atomic_read(&q->sq_out_pending));
+ }
+ }
+ /*
+ * System suspend is a preserved-state pause, not endpoint teardown.
+ * Deferred TRANSFER_REQUEST events already emitted by the T2 still belong
+ * to the same endpoint after resume; dropping them can make the first
+ * post-resume EP0 control URB wait until usbcore's timeout path.
+ */
+ if ((status = bce_vhci_cmd_endpoint_set_state(
+ &q->vhci->cq, q->dev_addr, endp_addr, BCE_VHCI_ENDPOINT_PAUSED, &q->state))) {
+ ret = status;
+ goto out;
+ }
+ if (q->state != BCE_VHCI_ENDPOINT_PAUSED) {
+ ret = -EINVAL;
+ goto out;
+ }
+ q->paused_by |= BCE_VHCI_PAUSE_SUSPEND;
+ pr_debug("t2bce_vhci: system suspend dev=%u ep=%02x path=firmware-pause paused_by=%x->%x ret=%d state=%x active=%u\n",
+ q->dev_addr, q->endp_addr, old_paused_by, q->paused_by,
+ ret, q->state, q->active);
+
+out:
+ mutex_unlock(&q->pause_lock);
+ return ret;
+}
+
+static void bce_vhci_transfer_queue_reset_w(struct work_struct *work)
+{
+ unsigned long flags;
+ struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_reset);
+
+ mutex_lock(&q->pause_lock);
+ spin_lock_irqsave(&q->urb_lock, flags);
+ pr_debug("t2bce_vhci: tq reset start dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+ if (!q->stalled) {
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ mutex_unlock(&q->pause_lock);
+ pr_debug("t2bce_vhci: tq reset skipped dev=%u port=%d ep=%02x stalled=0 active=%u paused_by=%x state=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->active, q->paused_by, q->state);
+ return;
+ }
+ q->active = false;
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ q->paused_by |= BCE_VHCI_PAUSE_INTERNAL_WQ;
+ bce_vhci_transfer_queue_remove_pending(q);
+ if (q->sq_in)
+ t2bce_core_flush_queue(q->vhci->client, q->sq_in);
+ if (q->sq_out)
+ t2bce_core_flush_queue(q->vhci->client, q->sq_out);
+ pr_debug("t2bce_vhci: tq reset command dev=%u port=%d ep=%02x\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr);
+ bce_vhci_cmd_endpoint_reset(&q->vhci->cq, q->dev_addr, (u8) (q->endp->desc.bEndpointAddress & 0x8F));
+ spin_lock_irqsave(&q->urb_lock, flags);
+ q->stalled = false;
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ mutex_unlock(&q->pause_lock);
+ bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ);
+ pr_debug("t2bce_vhci: tq reset done dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+}
+
+static void bce_vhci_transfer_queue_resume_w(struct work_struct *work)
+{
+ int ret;
+ struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_resume);
+
+ pr_debug("t2bce_vhci: tq async resume start dev=%u port=%d ep=%02x active=%u paused_by=%x stalled=%u state=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+ ret = bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ);
+ pr_debug("t2bce_vhci: tq async resume done dev=%u port=%d ep=%02x ret=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->endp_addr,
+ ret, q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests);
+}
+
+static void bce_vhci_transfer_queue_cancel_w(struct work_struct *work)
+{
+ unsigned long flags;
+ struct bce_vhci_transfer_queue *q = container_of(work, struct bce_vhci_transfer_queue, w_cancel);
+ struct bce_vhci_urb *vurb;
+ struct urb *urb;
+ int status;
+ bool was_active;
+
+ for (;;) {
+ spin_lock_irqsave(&q->urb_lock, flags);
+ if (list_empty(&q->cancel_urb_list)) {
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ return;
+ }
+
+ vurb = list_first_entry(&q->cancel_urb_list, struct bce_vhci_urb, cancel_list);
+ list_del_init(&vurb->cancel_list);
+ urb = vurb->urb;
+ status = vurb->cancel_status;
+ was_active = vurb->cancel_was_active;
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+
+ /* Endpoint pause talks to bridgeOS and may sleep. */
+ if (was_active) {
+ pr_debug("t2bce_vhci: [%02x] async cancelling URB\n", q->endp_addr);
+ bce_vhci_transfer_queue_pause(q, BCE_VHCI_PAUSE_INTERNAL_WQ);
+ }
+
+ spin_lock_irqsave(&q->urb_lock, flags);
+ if (was_active)
+ ++q->remaining_active_requests;
+ usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb);
+ urb->hcpriv = NULL;
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+
+ usb_hcd_giveback_urb(q->vhci->hcd, urb, status);
+
+ if (was_active)
+ bce_vhci_transfer_queue_resume(q, BCE_VHCI_PAUSE_INTERNAL_WQ);
+
+ bce_vhci_urb_free(vurb);
+ }
+}
+
+void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q)
+{
+ queue_work(q->vhci->tq_state_wq, &q->w_reset);
+}
+
+static void bce_vhci_transfer_queue_init_pending_urbs(struct bce_vhci_transfer_queue *q)
+{
+ struct urb *urb, *urbt;
+ struct bce_vhci_urb *vurb;
+ list_for_each_entry_safe(urb, urbt, &q->endp->urb_list, urb_list) {
+ vurb = urb->hcpriv;
+ if (!bce_vhci_transfer_queue_can_init_urb(q))
+ break;
+ if (vurb->state == BCE_VHCI_URB_INIT_PENDING)
+ bce_vhci_urb_init(vurb);
+ }
+}
+
+
+
+static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout);
+
+static bool bce_vhci_urb_uses_sg(struct bce_vhci_urb *urb)
+{
+ return urb->sg_segments && urb->urb->sg && urb->urb->num_mapped_sgs > 0;
+}
+
+static void bce_vhci_urb_sync_sg_segments_for_cpu(struct bce_vhci_urb *urb)
+{
+ struct device *dev;
+
+ if (!urb->sg_segments_for_device)
+ return;
+
+ dev = t2bce_core_client_dma_dev(urb->q->vhci->client);
+ dma_sync_single_for_cpu(dev, urb->sg_segments_dma, urb->sg_segments_size,
+ DMA_TO_DEVICE);
+ urb->sg_segments_for_device = false;
+}
+
+static void bce_vhci_urb_free(struct bce_vhci_urb *urb)
+{
+ struct device *dev;
+
+ if (!urb)
+ return;
+
+ if (urb->sg_segments) {
+ bce_vhci_urb_sync_sg_segments_for_cpu(urb);
+ dev = t2bce_core_client_dma_dev(urb->q->vhci->client);
+ dma_unmap_single(dev, urb->sg_segments_dma, urb->sg_segments_size,
+ DMA_TO_DEVICE);
+ kfree(urb->sg_segments);
+ }
+
+ kfree(urb);
+}
+
+static int bce_vhci_urb_init_sg_segments(struct bce_vhci_urb *vurb)
+{
+ struct device *dev;
+
+ if (!vurb->urb->sg || vurb->urb->num_mapped_sgs <= 0)
+ return 0;
+
+ if (vurb->urb->num_mapped_sgs > SIZE_MAX / sizeof(*vurb->sg_segments))
+ return -EOVERFLOW;
+
+ vurb->sg_segment_count = vurb->urb->num_mapped_sgs;
+ vurb->sg_segments_size = vurb->sg_segment_count * sizeof(*vurb->sg_segments);
+ vurb->sg_segments = kcalloc(vurb->sg_segment_count,
+ sizeof(*vurb->sg_segments), GFP_KERNEL);
+ if (!vurb->sg_segments)
+ return -ENOMEM;
+
+ dev = t2bce_core_client_dma_dev(vurb->q->vhci->client);
+ vurb->sg_segments_dma = dma_map_single(dev, vurb->sg_segments,
+ vurb->sg_segments_size, DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, vurb->sg_segments_dma)) {
+ kfree(vurb->sg_segments);
+ vurb->sg_segments = NULL;
+ vurb->sg_segments_size = 0;
+ vurb->sg_segment_count = 0;
+ return -ENOMEM;
+ }
+ vurb->sg_segments_for_device = true;
+ bce_vhci_urb_sync_sg_segments_for_cpu(vurb);
+
+ return 0;
+}
+
+static int bce_vhci_urb_set_sg_submission(struct bce_vhci_urb *urb,
+ struct t2bce_core_queue_sq *sq, u32 offset, u32 size)
+{
+ struct device *dev = t2bce_core_client_dma_dev(urb->q->vhci->client);
+ struct scatterlist *sg;
+ unsigned int i;
+ unsigned int segment_count = 0;
+ u32 skip = offset;
+ u32 remaining = size;
+
+ if (!bce_vhci_urb_uses_sg(urb) || !size)
+ return -EINVAL;
+
+ /*
+ * BCE consumes a compact address/length array, not Linux scatterlist
+ * metadata. Rebuild that array for the requested URB window.
+ */
+ bce_vhci_urb_sync_sg_segments_for_cpu(urb);
+
+ for_each_sg(urb->urb->sg, sg, urb->urb->num_mapped_sgs, i) {
+ dma_addr_t addr = sg_dma_address(sg);
+ u32 sg_len = sg_dma_len(sg);
+ u32 segment_len;
+
+ if (!sg_len)
+ continue;
+ if (skip >= sg_len) {
+ skip -= sg_len;
+ continue;
+ }
+
+ addr += skip;
+ sg_len -= skip;
+ skip = 0;
+
+ segment_len = min(sg_len, remaining);
+ urb->sg_segments[segment_count].addr = addr;
+ urb->sg_segments[segment_count].length = segment_len;
+ segment_count++;
+ remaining -= segment_len;
+
+ if (!remaining)
+ break;
+ }
+
+ if (remaining || !segment_count)
+ return -EINVAL;
+
+ dma_sync_single_for_device(dev, urb->sg_segments_dma,
+ segment_count * sizeof(*urb->sg_segments), DMA_TO_DEVICE);
+ urb->sg_segments_for_device = true;
+ t2bce_core_set_next_submission_segment_list(sq, urb->sg_segments_dma,
+ segment_count * sizeof(*urb->sg_segments));
+ return 0;
+}
+
+int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb)
+{
+ unsigned long flags;
+ int status = 0;
+ struct bce_vhci_urb *vurb;
+ bool is_control = (usb_endpoint_num(&urb->ep->desc) == 0);
+
+ if (is_control && q->endp_addr == 0x00 &&
+ !q->active && !q->stalled &&
+ q->paused_by == BCE_VHCI_PAUSE_INTERNAL_WQ) {
+ pr_warn("t2bce_vhci: EP0 enqueue scheduling async stale internal pause recovery dev=%u port=%d state=%u remaining=%u urb_reject=%d\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), q->state,
+ q->remaining_active_requests, atomic_read(&urb->reject));
+ queue_work(q->vhci->tq_state_wq, &q->w_resume);
+ }
+
+ vurb = kzalloc(sizeof(struct bce_vhci_urb), GFP_KERNEL);
+ if (!vurb)
+ return -ENOMEM;
+
+ urb->hcpriv = vurb;
+
+ vurb->q = q;
+ vurb->urb = urb;
+ vurb->dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+ vurb->is_control = is_control;
+ INIT_LIST_HEAD(&vurb->cancel_list);
+ status = bce_vhci_urb_init_sg_segments(vurb);
+ if (status) {
+ urb->hcpriv = NULL;
+ bce_vhci_urb_free(vurb);
+ return status;
+ }
+
+ bce_vhci_urb_log_control(vurb, "enqueue");
+
+ spin_lock_irqsave(&q->urb_lock, flags);
+ if (vurb->is_control && (!q->active || q->stalled || q->paused_by))
+ pr_debug("t2bce_vhci: EP0 enqueue dev=%u port=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u urb_reject=%d\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q),
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests, atomic_read(&urb->reject));
+
+ status = usb_hcd_link_urb_to_ep(q->vhci->hcd, urb);
+ if (status) {
+ if (vurb->is_control)
+ pr_warn("t2bce_vhci: EP0 enqueue link failed dev=%u port=%d status=%d active=%u paused_by=%x stalled=%u state=%u remaining=%u urb_reject=%d\n",
+ q->dev_addr, bce_vhci_transfer_queue_port(q), status,
+ q->active, q->paused_by, q->stalled, q->state,
+ q->remaining_active_requests, atomic_read(&urb->reject));
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ urb->hcpriv = NULL;
+ bce_vhci_urb_free(vurb);
+ return status;
+ }
+
+ if (q->active) {
+ if (bce_vhci_transfer_queue_can_init_urb(vurb->q))
+ status = bce_vhci_urb_init(vurb);
+ else
+ vurb->state = BCE_VHCI_URB_INIT_PENDING;
+ } else {
+ if (q->stalled)
+ bce_vhci_transfer_queue_request_reset(q);
+ vurb->state = BCE_VHCI_URB_INIT_PENDING;
+ }
+ if (status) {
+ usb_hcd_unlink_urb_from_ep(q->vhci->hcd, urb);
+ urb->hcpriv = NULL;
+ bce_vhci_urb_free(vurb);
+ } else {
+ bce_vhci_transfer_queue_deliver_pending(q);
+ }
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ pr_debug("t2bce_vhci: [%02x] URB enqueued (dir = %s, size = %i)\n", q->endp_addr,
+ usb_urb_dir_in(urb) ? "IN" : "OUT", urb->transfer_buffer_length);
+ return status;
+}
+
+static int bce_vhci_urb_init(struct bce_vhci_urb *vurb)
+{
+ int status = 0;
+
+ if (vurb->q->remaining_active_requests == 0) {
+ pr_err("t2bce_vhci: cannot init request (remaining_active_requests = 0)\n");
+ return -EINVAL;
+ }
+
+ if (vurb->is_control) {
+ vurb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST;
+ bce_vhci_urb_log_control(vurb, "init-wait-setup");
+ } else {
+ status = bce_vhci_urb_data_start(vurb, NULL);
+ }
+
+ if (!status) {
+ --vurb->q->remaining_active_requests;
+ }
+ return status;
+}
+
+static void bce_vhci_urb_complete(struct bce_vhci_urb *urb, int status)
+{
+ struct bce_vhci_transfer_queue *q = urb->q;
+ struct bce_vhci *vhci = q->vhci;
+ struct urb *real_urb = urb->urb;
+ pr_debug("t2bce_vhci: [%02x] URB complete %i\n", q->endp_addr, status);
+ bce_vhci_urb_log_control(urb, "complete");
+ usb_hcd_unlink_urb_from_ep(vhci->hcd, real_urb);
+ real_urb->hcpriv = NULL;
+ real_urb->status = status;
+ if (urb->state != BCE_VHCI_URB_INIT_PENDING)
+ ++urb->q->remaining_active_requests;
+ bce_vhci_urb_free(urb);
+ list_add_tail(&real_urb->urb_list, &q->giveback_urb_list);
+}
+
+int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status)
+{
+ struct bce_vhci_urb *vurb;
+ unsigned long flags;
+ int ret;
+ enum bce_vhci_urb_state old_state;
+
+ spin_lock_irqsave(&q->urb_lock, flags);
+ if ((ret = usb_hcd_check_unlink_urb(q->vhci->hcd, urb, status))) {
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ return ret;
+ }
+
+ vurb = urb->hcpriv;
+ if (!vurb) {
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ return -ENOENT;
+ }
+
+ old_state = vurb->state;
+ bce_vhci_urb_log_control(vurb, "cancel-request");
+
+ if (old_state == BCE_VHCI_URB_CANCELLED) {
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ pr_debug("t2bce_vhci: URB %p is already cancelled, skipping\n", urb);
+ return 0;
+ }
+
+ vurb->state = BCE_VHCI_URB_CANCELLED;
+ vurb->cancel_status = status;
+ vurb->cancel_was_active = old_state != BCE_VHCI_URB_INIT_PENDING;
+
+ if (list_empty(&vurb->cancel_list))
+ list_add_tail(&vurb->cancel_list, &q->cancel_urb_list);
+ spin_unlock_irqrestore(&q->urb_lock, flags);
+ queue_work(q->vhci->tq_state_wq, &q->w_cancel);
+
+ return 0;
+}
+
+static int bce_vhci_urb_data_transfer_in(struct bce_vhci_urb *urb, unsigned long *timeout)
+{
+ struct bce_vhci_message msg;
+ u32 tr_len;
+ int reservation1, reservation2 = -EFAULT;
+ bool uses_sg;
+ int status;
+
+ pr_debug("t2bce_vhci: [%02x] DMA from device %llx %x\n", urb->q->endp_addr,
+ (u64) urb->urb->transfer_dma, urb->urb->transfer_buffer_length);
+
+ /* The firmware request and the inbound DMA submission must be queued as a pair. */
+ reservation1 = t2bce_core_reserve_submission(urb->q->vhci->msg_asynchronous.sq, timeout);
+ if (!reservation1)
+ reservation2 = t2bce_core_reserve_submission(urb->q->sq_in, timeout);
+ if (reservation1 || reservation2) {
+ pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n");
+ if (!reservation1)
+ t2bce_core_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq);
+ return -ENOMEM;
+ }
+
+ urb->send_offset = urb->receive_offset;
+
+ tr_len = urb->urb->transfer_buffer_length - urb->send_offset;
+ uses_sg = bce_vhci_urb_uses_sg(urb);
+
+ if (uses_sg) {
+ status = bce_vhci_urb_set_sg_submission(urb, urb->q->sq_in,
+ urb->send_offset, tr_len);
+ if (status) {
+ t2bce_core_cancel_submission_reservation(urb->q->sq_in);
+ t2bce_core_cancel_submission_reservation(urb->q->vhci->msg_asynchronous.sq);
+ pr_err("t2bce_vhci: [%02x] failed to prepare SG IN submission: %d\n",
+ urb->q->endp_addr, status);
+ return status;
+ }
+ }
+
+ spin_lock(&urb->q->vhci->msg_asynchronous_lock);
+ msg.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
+ msg.status = 0;
+ msg.param1 = ((urb->urb->ep->desc.bEndpointAddress & 0x8Fu) << 8) | urb->q->dev_addr;
+ msg.param2 = tr_len;
+ bce_vhci_message_queue_write(&urb->q->vhci->msg_asynchronous, &msg);
+ spin_unlock(&urb->q->vhci->msg_asynchronous_lock);
+
+ if (!uses_sg)
+ t2bce_core_set_next_submission_single(urb->q->sq_in,
+ urb->urb->transfer_dma + urb->send_offset, tr_len);
+ t2bce_core_submit_to_device(urb->q->sq_in);
+
+ urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION;
+ return 0;
+}
+
+static int bce_vhci_urb_data_start(struct bce_vhci_urb *urb, unsigned long *timeout)
+{
+ if (urb->dir == DMA_TO_DEVICE) {
+ if (urb->urb->transfer_buffer_length > 0)
+ urb->state = BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST;
+ else
+ urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE;
+ bce_vhci_urb_log_control(urb, "data-start-out");
+ return 0;
+ } else {
+ bce_vhci_urb_log_control(urb, "data-start-in");
+ return bce_vhci_urb_data_transfer_in(urb, timeout);
+ }
+}
+
+static int bce_vhci_urb_send_out_data(struct bce_vhci_urb *urb, dma_addr_t addr, size_t size)
+{
+ unsigned long timeout = 0;
+ if (t2bce_core_reserve_submission(urb->q->sq_out, &timeout)) {
+ pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n");
+ return -EPIPE;
+ }
+
+ pr_debug("t2bce_vhci: [%02x] DMA to device %llx %lx\n", urb->q->endp_addr, (u64) addr, size);
+
+ t2bce_core_set_next_submission_single(urb->q->sq_out, addr, size);
+ atomic_inc(&urb->q->sq_out_pending);
+ t2bce_core_submit_to_device(urb->q->sq_out);
+ return 0;
+}
+
+static int bce_vhci_urb_send_out_transfer_data(struct bce_vhci_urb *urb,
+ u32 offset, size_t size)
+{
+ unsigned long timeout = 0;
+ int status;
+
+ if (t2bce_core_reserve_submission(urb->q->sq_out, &timeout)) {
+ pr_err("t2bce_vhci: Failed to reserve a submission for URB data transfer\n");
+ return -EPIPE;
+ }
+
+ if (bce_vhci_urb_uses_sg(urb)) {
+ status = bce_vhci_urb_set_sg_submission(urb, urb->q->sq_out,
+ offset, size);
+ if (status) {
+ t2bce_core_cancel_submission_reservation(urb->q->sq_out);
+ pr_err("t2bce_vhci: [%02x] failed to prepare SG OUT submission: %d\n",
+ urb->q->endp_addr, status);
+ return status;
+ }
+ } else {
+ t2bce_core_set_next_submission_single(urb->q->sq_out,
+ urb->urb->transfer_dma + offset, size);
+ }
+
+ atomic_inc(&urb->q->sq_out_pending);
+ t2bce_core_submit_to_device(urb->q->sq_out);
+ return 0;
+}
+
+static int bce_vhci_urb_data_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg)
+{
+ u32 tr_len;
+ int status;
+ if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST) {
+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) {
+ tr_len = min(urb->urb->transfer_buffer_length - urb->send_offset, (u32) msg->param2);
+ bce_vhci_urb_log_control(urb, "data-request");
+ if ((status = bce_vhci_urb_send_out_transfer_data(urb, urb->send_offset, tr_len)))
+ return status;
+ urb->send_offset += tr_len;
+ urb->state = BCE_VHCI_URB_WAITING_FOR_COMPLETION;
+ bce_vhci_urb_log_control(urb, "data-sent");
+ return 0;
+ }
+ }
+
+ /* OUT endpoints can receive extra transfer requests while waiting for host data. */
+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL)
+ return -EAGAIN;
+ pr_err("t2bce_vhci: [%02x] %s URB unexpected message (state = %x, msg: %x %x %x %llx)\n",
+ urb->q->endp_addr, (urb->is_control ? "Control (data update)" : "Data"), urb->state,
+ msg->cmd, msg->status, msg->param1, msg->param2);
+ return -EAGAIN;
+}
+
+static int bce_vhci_urb_data_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c)
+{
+ bce_vhci_urb_sync_sg_segments_for_cpu(urb);
+
+ if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) {
+ urb->receive_offset += c->data_size;
+ if (urb->dir == DMA_FROM_DEVICE || urb->receive_offset >= urb->urb->transfer_buffer_length) {
+ urb->urb->actual_length = (u32) urb->receive_offset;
+ urb->state = BCE_VHCI_URB_DATA_TRANSFER_COMPLETE;
+ bce_vhci_urb_log_control(urb, "data-complete");
+ if (!urb->is_control) {
+ bce_vhci_urb_complete(urb, 0);
+ return -ENOENT;
+ }
+ }
+ } else {
+ pr_warn_ratelimited("t2bce_vhci: [%02x] data URB unexpected completion state=%x status=%x size=%llu\n",
+ urb->q->endp_addr, urb->state, c->status, c->data_size);
+ }
+ return 0;
+}
+
+
+static int bce_vhci_urb_control_check_status(struct bce_vhci_urb *urb)
+{
+ struct bce_vhci_transfer_queue *q = urb->q;
+ if (urb->received_status == 0)
+ return 0;
+ if (urb->state == BCE_VHCI_URB_DATA_TRANSFER_COMPLETE ||
+ (urb->received_status != BCE_VHCI_SUCCESS && urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST &&
+ urb->state != BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION)) {
+ urb->state = BCE_VHCI_URB_CONTROL_COMPLETE;
+ if (urb->received_status != BCE_VHCI_SUCCESS) {
+ if (urb->received_status == 3 && urb->q->endp_addr == 0x00) {
+ q->active = false;
+ q->stalled = true;
+ q->state = BCE_VHCI_ENDPOINT_STALLED;
+ pr_debug("t2bce_vhci: EP0 status=3 dev=%u state=%x; completing urb and scheduling endpoint reset paused_by=%x remaining=%u\n",
+ q->dev_addr, urb->state, q->paused_by,
+ q->remaining_active_requests);
+ bce_vhci_urb_complete(urb, -EPIPE);
+ bce_vhci_transfer_queue_request_reset(q);
+ return -ENOENT;
+ }
+
+ pr_err("t2bce_vhci: [%02x] URB failed: %x\n",
+ urb->q->endp_addr, urb->received_status);
+ urb->q->active = false;
+ urb->q->stalled = true;
+ bce_vhci_urb_complete(urb, -EPIPE);
+ if (!list_empty(&q->endp->urb_list))
+ bce_vhci_transfer_queue_request_reset(q);
+ return -ENOENT;
+ }
+ bce_vhci_urb_complete(urb, 0);
+ return -ENOENT;
+ }
+ return 0;
+}
+
+static int bce_vhci_urb_control_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg)
+{
+ int status;
+ if (msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) {
+ urb->received_status = msg->status;
+ bce_vhci_urb_log_control(urb, "status-msg");
+ return bce_vhci_urb_control_check_status(urb);
+ }
+
+ if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST) {
+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST) {
+ bce_vhci_urb_log_control(urb, "setup-request");
+ if (bce_vhci_urb_send_out_data(urb, urb->urb->setup_dma, sizeof(struct usb_ctrlrequest))) {
+ pr_err("t2bce_vhci: [%02x] Failed to start URB setup transfer\n", urb->q->endp_addr);
+ /* Keep the old state machine behavior; endpoint recovery is driven by later firmware status. */
+ return 0;
+ }
+ urb->state = BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION;
+ bce_vhci_urb_log_control(urb, "setup-sent");
+ pr_debug("t2bce_vhci: [%02x] Sent setup %llx\n", urb->q->endp_addr, urb->urb->setup_dma);
+ return 0;
+ }
+ } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST ||
+ urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) {
+ if ((status = bce_vhci_urb_data_update(urb, msg)))
+ return status;
+ return bce_vhci_urb_control_check_status(urb);
+ }
+
+ /* OUT endpoints can receive extra transfer requests while waiting for host data. */
+ if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST && urb->q->sq_out != NULL)
+ return -EAGAIN;
+ pr_err("t2bce_vhci: [%02x] Control URB unexpected message (state = %x, msg: %x %x %x %llx)\n", urb->q->endp_addr,
+ urb->state, msg->cmd, msg->status, msg->param1, msg->param2);
+ return -EAGAIN;
+}
+
+static int bce_vhci_urb_control_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c)
+{
+ int status;
+ unsigned long timeout;
+
+ if (urb->state == BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION) {
+ if (c->data_size != sizeof(struct usb_ctrlrequest))
+ pr_err("t2bce_vhci: [%02x] transfer complete data size mistmatch for usb_ctrlrequest (%llx instead of %lx)\n",
+ urb->q->endp_addr, c->data_size, sizeof(struct usb_ctrlrequest));
+
+ bce_vhci_urb_log_control(urb, "setup-complete");
+ timeout = 1000;
+ status = bce_vhci_urb_data_start(urb, &timeout);
+ if (status) {
+ bce_vhci_urb_complete(urb, status);
+ return -ENOENT;
+ }
+ return 0;
+ } else if (urb->state == BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST ||
+ urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) {
+ if ((status = bce_vhci_urb_data_transfer_completion(urb, c)))
+ return status;
+ return bce_vhci_urb_control_check_status(urb);
+ } else {
+ pr_err("t2bce_vhci: [%02x] Control URB unexpected completion (state = %x)\n", urb->q->endp_addr, urb->state);
+ }
+ return 0;
+}
+
+static int bce_vhci_urb_update(struct bce_vhci_urb *urb, struct bce_vhci_message *msg)
+{
+ if (urb->state == BCE_VHCI_URB_INIT_PENDING)
+ return -EAGAIN;
+ if (urb->is_control)
+ return bce_vhci_urb_control_update(urb, msg);
+ else
+ return bce_vhci_urb_data_update(urb, msg);
+}
+
+static int bce_vhci_urb_transfer_completion(struct bce_vhci_urb *urb, struct t2bce_core_sq_completion_data *c)
+{
+ if (urb->is_control)
+ return bce_vhci_urb_control_transfer_completion(urb, c);
+ else
+ return bce_vhci_urb_data_transfer_completion(urb, c);
+}
+
+static void bce_vhci_urb_resume(struct bce_vhci_urb *urb)
+{
+ int status = 0;
+ if (urb->state == BCE_VHCI_URB_WAITING_FOR_COMPLETION) {
+ status = bce_vhci_urb_data_transfer_in(urb, NULL);
+ }
+ if (status)
+ bce_vhci_urb_complete(urb, status);
+}
diff --git a/drivers/staging/t2bce/t2bce_vhci/transfer.h b/drivers/staging/t2bce/t2bce_vhci/transfer.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/transfer.h
@@ -0,0 +1,86 @@
+#ifndef BCEDRIVER_TRANSFER_H
+#define BCEDRIVER_TRANSFER_H
+
+#include <linux/wait.h>
+#include <linux/usb.h>
+#include "queue.h"
+#include "command.h"
+
+struct bce_vhci_list_message {
+ struct list_head list;
+ struct bce_vhci_message msg;
+};
+enum bce_vhci_pause_source {
+ BCE_VHCI_PAUSE_INTERNAL_WQ = 1,
+ BCE_VHCI_PAUSE_FIRMWARE = 2,
+ BCE_VHCI_PAUSE_SUSPEND = 4,
+ BCE_VHCI_PAUSE_SHUTDOWN = 8
+};
+struct bce_vhci_transfer_queue {
+ struct bce_vhci *vhci;
+ struct usb_host_endpoint *endp;
+ enum bce_vhci_endpoint_state state;
+ u32 max_active_requests, remaining_active_requests;
+ bool active, stalled;
+ u32 paused_by;
+ bce_vhci_device_t dev_addr;
+ u8 endp_addr;
+ struct t2bce_core_queue_cq *cq;
+ struct t2bce_core_queue_sq *sq_in;
+ struct t2bce_core_queue_sq *sq_out;
+ struct list_head evq;
+ struct spinlock urb_lock;
+ struct mutex pause_lock;
+ struct list_head giveback_urb_list;
+ wait_queue_head_t sq_out_wait_queue;
+ atomic_t sq_out_pending;
+ struct list_head cancel_urb_list;
+
+ struct work_struct w_reset;
+ struct work_struct w_resume;
+ struct work_struct w_cancel;
+};
+enum bce_vhci_urb_state {
+ BCE_VHCI_URB_INIT_PENDING,
+ BCE_VHCI_URB_WAITING_FOR_TRANSFER_REQUEST,
+ BCE_VHCI_URB_WAITING_FOR_COMPLETION,
+ BCE_VHCI_URB_DATA_TRANSFER_COMPLETE,
+ BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_REQUEST,
+ BCE_VHCI_URB_CONTROL_WAITING_FOR_SETUP_COMPLETION,
+ BCE_VHCI_URB_CONTROL_COMPLETE,
+ BCE_VHCI_URB_CANCELLED
+};
+struct bce_vhci_urb {
+ struct urb *urb;
+ struct bce_vhci_transfer_queue *q;
+ enum dma_data_direction dir;
+ bool is_control;
+ enum bce_vhci_urb_state state;
+ int received_status;
+ u32 send_offset;
+ u32 receive_offset;
+ struct t2bce_core_dma_segment *sg_segments;
+ dma_addr_t sg_segments_dma;
+ size_t sg_segments_size;
+ unsigned int sg_segment_count;
+ bool sg_segments_for_device;
+ struct list_head cancel_list;
+ int cancel_status;
+ bool cancel_was_active;
+};
+
+void bce_vhci_create_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q,
+ struct usb_host_endpoint *endp, bce_vhci_device_t dev_addr, enum dma_data_direction dir);
+void bce_vhci_destroy_transfer_queue(struct bce_vhci *vhci, struct bce_vhci_transfer_queue *q);
+void bce_vhci_transfer_queue_event(struct bce_vhci_transfer_queue *q, struct bce_vhci_message *msg);
+int bce_vhci_transfer_queue_do_pause(struct bce_vhci_transfer_queue *q);
+int bce_vhci_transfer_queue_do_resume(struct bce_vhci_transfer_queue *q);
+int bce_vhci_transfer_queue_pause(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src);
+int bce_vhci_transfer_queue_resume(struct bce_vhci_transfer_queue *q, enum bce_vhci_pause_source src);
+int bce_vhci_transfer_queue_suspend_pause(struct bce_vhci_transfer_queue *q);
+void bce_vhci_transfer_queue_request_reset(struct bce_vhci_transfer_queue *q);
+
+int bce_vhci_urb_create(struct bce_vhci_transfer_queue *q, struct urb *urb);
+int bce_vhci_urb_request_cancel(struct bce_vhci_transfer_queue *q, struct urb *urb, int status);
+
+#endif //BCEDRIVER_TRANSFER_H
diff --git a/drivers/staging/t2bce/t2bce_vhci/vhci.c b/drivers/staging/t2bce/t2bce_vhci/vhci.c
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/vhci.c
@@ -0,0 +1,1293 @@
+#include "vhci.h"
+#include "command.h"
+#include <linux/delay.h>
+#include <linux/usb.h>
+#include <linux/usb/hcd.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/err.h>
+#include <linux/slab.h>
+
+/*
+ * bridgeOS exposes several T2-internal devices through a virtual USB 2.0 hub.
+ * This HCD translates usbcore hub, device and endpoint operations into the
+ * firmware command/event protocol carried over BCE queues.
+ */
+
+static dev_t bce_vhci_chrdev;
+static struct class *bce_vhci_class;
+static const struct hc_driver bce_vhci_driver;
+static u16 bce_vhci_port_mask = U16_MAX;
+static struct bce_vhci *global_vhci;
+
+enum bce_vhci_port_status_bits {
+ BCE_VHCI_PORT_STATUS_OVERCURRENT = 0x2,
+ BCE_VHCI_PORT_STATUS_CONNECTED = 0x4,
+ BCE_VHCI_PORT_STATUS_RESET = 0x8,
+ BCE_VHCI_PORT_STATUS_ENABLED = 0x10,
+ BCE_VHCI_PORT_STATUS_SUSPENDED_LEGACY = 0x60,
+ BCE_VHCI_PORT_STATUS_SUSPENDED = 0x80,
+ BCE_VHCI_PORT_STATUS_HIGH_SPEED = 0x200,
+ BCE_VHCI_PORT_STATUS_C_CONNECTION = 0x40000,
+};
+
+static int bce_vhci_create_event_queues(struct bce_vhci *vhci);
+static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci);
+static int bce_vhci_create_message_queues(struct bce_vhci *vhci);
+static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci);
+static void bce_vhci_handle_firmware_events_w(struct work_struct *ws);
+static void bce_vhci_add_hcd_w(struct work_struct *ws);
+static void bce_vhci_firmware_event_completion(struct t2bce_core_queue_sq *sq);
+static int bce_vhci_start_controller(struct bce_vhci *vhci);
+static void bce_vhci_forget_devices(struct bce_vhci *vhci);
+static int __bce_vhci_add_hcd(struct bce_vhci *vhci);
+static void bce_vhci_shutdown_client(void *userdata);
+static void bce_vhci_pm_reset_client(void *userdata);
+static void bce_vhci_pm_prepare_client(void *userdata);
+static void bce_vhci_pm_prepare_no_state_client(void *userdata);
+static void bce_vhci_pm_mark_no_state_resume_client(void *userdata);
+static bool bce_vhci_pm_is_no_state_resume_client(void *userdata);
+static void bce_vhci_pm_complete_client(void *userdata);
+
+static const struct t2bce_core_client_pm_ops bce_vhci_pm_ops = {
+ .shutdown = bce_vhci_shutdown_client,
+ .pm_reset = bce_vhci_pm_reset_client,
+ .pm_prepare = bce_vhci_pm_prepare_client,
+ .pm_prepare_no_state = bce_vhci_pm_prepare_no_state_client,
+ .pm_mark_no_state_resume = bce_vhci_pm_mark_no_state_resume_client,
+ .pm_is_no_state_resume = bce_vhci_pm_is_no_state_resume_client,
+ .pm_complete = bce_vhci_pm_complete_client,
+};
+
+int bce_vhci_create(struct device *parent, struct bce_vhci *vhci)
+{
+ int status;
+
+ spin_lock_init(&vhci->hcd_spinlock);
+
+ vhci->vdevt = bce_vhci_chrdev;
+ vhci->vdev = device_create(bce_vhci_class, parent, vhci->vdevt, NULL, "t2bce_vhci");
+ if (IS_ERR_OR_NULL(vhci->vdev)) {
+ status = PTR_ERR(vhci->vdev);
+ goto fail_dev;
+ }
+ vhci->client = t2bce_core_client_get(vhci->vdev);
+ if (IS_ERR(vhci->client)) {
+ status = PTR_ERR(vhci->client);
+ vhci->client = NULL;
+ goto fail_client;
+ }
+ t2bce_core_client_set_pm_ops(vhci->client, &bce_vhci_pm_ops, vhci);
+
+ if ((status = bce_vhci_create_message_queues(vhci)))
+ goto fail_mq;
+ if ((status = bce_vhci_create_event_queues(vhci)))
+ goto fail_eq;
+
+ vhci->tq_state_wq = alloc_ordered_workqueue("t2bce_vhci-tq-state", 0);
+ INIT_WORK(&vhci->w_fw_events, bce_vhci_handle_firmware_events_w);
+ INIT_WORK(&vhci->w_add_hcd, bce_vhci_add_hcd_w);
+ vhci->port_change_pending = 0;
+ vhci->stateful_suspended_bounce = 0;
+ vhci->stateful_resume = false;
+
+ vhci->hcd = usb_create_hcd(&bce_vhci_driver, vhci->vdev, "t2bce_vhci");
+ if (!vhci->hcd) {
+ status = -ENOMEM;
+ goto fail_hcd;
+ }
+ vhci->hcd->self.sysdev = t2bce_core_client_dma_dev(vhci->client);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
+ vhci->hcd->self.uses_dma = 1;
+#endif
+ *((struct bce_vhci **) vhci->hcd->hcd_priv) = vhci;
+ vhci->hcd->speed = HCD_USB2;
+
+ if ((status = __bce_vhci_add_hcd(vhci)))
+ goto fail_hcd;
+
+ pr_info("t2bce_vhci: initialized\n");
+ return 0;
+
+fail_hcd:
+ bce_vhci_destroy_event_queues(vhci);
+fail_eq:
+ bce_vhci_destroy_message_queues(vhci);
+fail_mq:
+ t2bce_core_client_set_pm_ops(vhci->client, NULL, NULL);
+ t2bce_core_client_put(vhci->client);
+ vhci->client = NULL;
+fail_client:
+ device_destroy(bce_vhci_class, vhci->vdevt);
+fail_dev:
+ if (!status)
+ status = -EINVAL;
+ return status;
+}
+
+void bce_vhci_destroy(struct bce_vhci *vhci)
+{
+ cancel_work_sync(&vhci->w_add_hcd);
+ bce_vhci_remove_hcd(vhci);
+ bce_vhci_destroy_event_queues(vhci);
+ bce_vhci_destroy_message_queues(vhci);
+ t2bce_core_client_set_pm_ops(vhci->client, NULL, NULL);
+ t2bce_core_client_put(vhci->client);
+ vhci->client = NULL;
+ device_destroy(bce_vhci_class, vhci->vdevt);
+}
+
+static int __bce_vhci_add_hcd(struct bce_vhci *vhci)
+{
+ int status;
+
+ status = usb_add_hcd(vhci->hcd, 0, 0);
+ if (!status)
+ vhci->hcd_registered = true;
+ return status;
+}
+
+int bce_vhci_add_hcd(struct bce_vhci *vhci)
+{
+ if (vhci->hcd_registered)
+ return 0;
+ return __bce_vhci_add_hcd(vhci);
+}
+
+void bce_vhci_remove_hcd(struct bce_vhci *vhci)
+{
+ cancel_work_sync(&vhci->w_add_hcd);
+ if (!vhci->hcd_registered)
+ return;
+ usb_remove_hcd(vhci->hcd);
+ vhci->hcd_registered = false;
+}
+
+void bce_vhci_shutdown(struct bce_vhci *vhci)
+{
+ cancel_work_sync(&vhci->w_add_hcd);
+ if (vhci->hcd_registered) {
+ bce_vhci_remove_hcd(vhci);
+ return;
+ }
+
+ /*
+ * Shutdown is not remove: the USB core may already have detached the HCD,
+ * but the T2-side controller can still be left enabled. Send the controller
+ * disable directly as a final lights-out command while queues still exist.
+ */
+ bce_vhci_cmd_controller_disable(&vhci->cq);
+}
+
+void bce_vhci_pm_reset(struct bce_vhci *vhci)
+{
+ vhci->no_state_resume = false;
+}
+
+void bce_vhci_pm_prepare(struct bce_vhci *vhci)
+{
+ cancel_work_sync(&vhci->w_add_hcd);
+ vhci->port_change_pending = 0;
+}
+
+void bce_vhci_pm_prepare_no_state(struct bce_vhci *vhci)
+{
+ bce_vhci_remove_hcd(vhci);
+}
+
+void bce_vhci_pm_mark_no_state_resume(struct bce_vhci *vhci)
+{
+ vhci->no_state_resume = true;
+}
+
+bool bce_vhci_pm_is_no_state_resume(struct bce_vhci *vhci)
+{
+ return vhci->no_state_resume;
+}
+
+void bce_vhci_pm_complete(struct bce_vhci *vhci)
+{
+ if (!vhci->no_state_resume)
+ return;
+
+ /* Re-add the VHCI HCD after the PM core completed resume ordering. */
+ pr_debug("t2bce_vhci: scheduling HCD re-add after no-state wake\n");
+ queue_work(vhci->tq_state_wq, &vhci->w_add_hcd);
+}
+
+static void bce_vhci_pm_reset_client(void *userdata)
+{
+ bce_vhci_pm_reset(userdata);
+}
+
+static void bce_vhci_pm_prepare_client(void *userdata)
+{
+ bce_vhci_pm_prepare(userdata);
+}
+
+static void bce_vhci_shutdown_client(void *userdata)
+{
+ bce_vhci_shutdown(userdata);
+}
+
+static void bce_vhci_pm_prepare_no_state_client(void *userdata)
+{
+ bce_vhci_pm_prepare_no_state(userdata);
+}
+
+static void bce_vhci_pm_mark_no_state_resume_client(void *userdata)
+{
+ bce_vhci_pm_mark_no_state_resume(userdata);
+}
+
+static bool bce_vhci_pm_is_no_state_resume_client(void *userdata)
+{
+ return bce_vhci_pm_is_no_state_resume(userdata);
+}
+
+static void bce_vhci_pm_complete_client(void *userdata)
+{
+ bce_vhci_pm_complete(userdata);
+}
+
+static void bce_vhci_add_hcd_w(struct work_struct *ws)
+{
+ struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_add_hcd);
+ int status;
+
+ pr_debug("t2bce_vhci: deferred HCD add after no-state wake\n");
+ status = bce_vhci_add_hcd(vhci);
+ if (status) {
+ pr_err("t2bce_vhci: deferred HCD add failed: %d\n", status);
+ return;
+ }
+
+ vhci->no_state_resume = false;
+}
+
+struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd)
+{
+ return *((struct bce_vhci **) hcd->hcd_priv);
+}
+
+int bce_vhci_start(struct usb_hcd *hcd)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ return bce_vhci_start_controller(vhci);
+}
+
+void bce_vhci_stop(struct usb_hcd *hcd)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ bce_vhci_cmd_controller_disable(&vhci->cq);
+}
+
+static int bce_vhci_start_controller(struct bce_vhci *vhci)
+{
+ int status;
+ u16 port_mask = 0;
+ bce_vhci_port_t port_no = 0;
+
+ if ((status = bce_vhci_cmd_controller_enable(&vhci->cq, 1, &port_mask)))
+ return status;
+ vhci->port_mask = port_mask;
+ vhci->port_power_mask = 0;
+ if ((status = bce_vhci_cmd_controller_start(&vhci->cq)))
+ return status;
+ while (port_mask) {
+ port_no += 1;
+ port_mask >>= 1;
+ }
+ vhci->port_count = port_no;
+ return 0;
+}
+
+static void bce_vhci_forget_devices(struct bce_vhci *vhci)
+{
+ int i, j;
+ bce_vhci_device_t devid;
+ struct bce_vhci_device *dev;
+
+ for (i = 0; i < 16; i++) {
+ devid = vhci->port_to_device[i];
+ if (!devid)
+ continue;
+
+ dev = vhci->devices[devid];
+ if (!dev) {
+ vhci->port_to_device[i] = 0;
+ continue;
+ }
+
+ for (j = 0; j < 32; j++) {
+ if (!(dev->tq_mask & BIT(j)))
+ continue;
+ if (dev->tq[j].endp)
+ dev->tq[j].endp->hcpriv = NULL;
+ bce_vhci_destroy_transfer_queue(vhci, &dev->tq[j]);
+ }
+
+ dev->tq_mask = 0;
+ vhci->devices[devid] = NULL;
+ vhci->port_to_device[i] = 0;
+ kfree(dev);
+ }
+}
+
+static int bce_vhci_hub_status_data(struct usb_hcd *hcd, char *buf)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ unsigned long pending;
+
+ /* Only ports reported by firmware as changed should wake the hub state machine. */
+ pending = READ_ONCE(vhci->port_change_pending);
+ if (!pending)
+ return 0;
+
+ memset(buf, 0, 2);
+ buf[0] = (char)((pending << 1) & 0xff);
+ buf[1] = (char)((pending >> 7) & 0xff);
+ return buf[0] || buf[1];
+}
+
+static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout);
+
+static int bce_vhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex, char *buf, u16 wLength)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ int status;
+ struct usb_hub_descriptor *hd;
+ struct usb_hub_status *hs;
+ struct usb_port_status *ps;
+ u32 port_status;
+ u32 raw_port_status;
+ if (typeReq == GetHubDescriptor && wLength >= sizeof(struct usb_hub_descriptor)) {
+ hd = (struct usb_hub_descriptor *) buf;
+ memset(hd, 0, sizeof(*hd));
+ hd->bDescLength = sizeof(struct usb_hub_descriptor);
+ hd->bDescriptorType = USB_DT_HUB;
+ hd->bNbrPorts = (u8) vhci->port_count;
+ hd->wHubCharacteristics = HUB_CHAR_INDV_PORT_LPSM | HUB_CHAR_INDV_PORT_OCPM;
+ hd->bPwrOn2PwrGood = 0;
+ hd->bHubContrCurrent = 0;
+ return 0;
+ } else if (typeReq == GetHubStatus && wLength >= sizeof(struct usb_hub_status)) {
+ hs = (struct usb_hub_status *) buf;
+ memset(hs, 0, sizeof(*hs));
+ hs->wHubStatus = 0;
+ hs->wHubChange = 0;
+ return 0;
+ } else if (typeReq == GetPortStatus && wLength >= 4 /* usb 2.0 */) {
+ ps = (struct usb_port_status *) buf;
+ ps->wPortStatus = 0;
+ ps->wPortChange = 0;
+
+ if (vhci->port_power_mask & BIT(wIndex))
+ ps->wPortStatus |= USB_PORT_STAT_POWER;
+
+ if (!(bce_vhci_port_mask & BIT(wIndex)))
+ return 0;
+
+ if ((status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0, &port_status)))
+ return status;
+ raw_port_status = port_status;
+
+ /*
+ * During stateful resume the T2 can briefly report a disconnect bounce
+ * for an already known virtual port. Keep usbcore from treating that
+ * transient as a real unplug; the follow-up status/event decides the
+ * actual port state.
+ */
+ if (vhci->stateful_resume &&
+ (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION) &&
+ !(port_status & BCE_VHCI_PORT_STATUS_CONNECTED) &&
+ wIndex < ARRAY_SIZE(vhci->port_to_device) &&
+ vhci->port_to_device[wIndex]) {
+ port_status |= BCE_VHCI_PORT_STATUS_CONNECTED |
+ BCE_VHCI_PORT_STATUS_ENABLED |
+ BCE_VHCI_PORT_STATUS_SUSPENDED;
+ pr_debug("t2bce_vhci: hub masked stateful disconnect bounce port=%u raw=%x masked_raw=%x\n",
+ wIndex, raw_port_status, port_status);
+ }
+ if (port_status & BCE_VHCI_PORT_STATUS_ENABLED)
+ ps->wPortStatus |= USB_PORT_STAT_ENABLE;
+ if (port_status & BCE_VHCI_PORT_STATUS_CONNECTED)
+ ps->wPortStatus |= USB_PORT_STAT_CONNECTION;
+ if (port_status & BCE_VHCI_PORT_STATUS_OVERCURRENT)
+ ps->wPortStatus |= USB_PORT_STAT_OVERCURRENT;
+ if (port_status & BCE_VHCI_PORT_STATUS_RESET)
+ ps->wPortStatus |= USB_PORT_STAT_RESET;
+ if (port_status & BCE_VHCI_PORT_STATUS_HIGH_SPEED)
+ ps->wPortStatus |= USB_PORT_STAT_HIGH_SPEED;
+ if (port_status & (BCE_VHCI_PORT_STATUS_SUSPENDED |
+ BCE_VHCI_PORT_STATUS_SUSPENDED_LEGACY)) {
+ ps->wPortStatus |= USB_PORT_STAT_SUSPEND;
+ if (vhci->stateful_resume && (port_status & BCE_VHCI_PORT_STATUS_CONNECTED))
+ ps->wPortStatus |= USB_PORT_STAT_ENABLE;
+ }
+
+ /*
+ * These ports are fixed T2-internal devices, not user-pluggable ports.
+ * For stateful resume, suppress firmware connection-change bounces so
+ * usbcore does not re-enumerate solely because the T2 toggled its
+ * internal representation while restoring state.
+ */
+ if (vhci->stateful_resume && (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION)) {
+ u32 cleared_status = 0;
+
+ clear_bit(wIndex - 1, &vhci->port_change_pending);
+ status = bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &cleared_status);
+ pr_debug("t2bce_vhci: hub suppressed C_CONNECTION port=%u raw=%x clear_status=%d cleared_raw=%x\n",
+ wIndex, port_status, status, cleared_status);
+ } else if (port_status & BCE_VHCI_PORT_STATUS_C_CONNECTION) {
+ ps->wPortChange |= USB_PORT_STAT_C_CONNECTION;
+ }
+ pr_debug("t2bce_vhci: hub GetPortStatus port=%u raw=%x status=%x change=%x\n",
+ wIndex, port_status, ps->wPortStatus, ps->wPortChange);
+ return 0;
+ } else if (typeReq == SetPortFeature) {
+ if (wValue == USB_PORT_FEAT_POWER) {
+ status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) wIndex);
+ /* usbcore tracks logical power state separately from bridgeOS. */
+ if (!status)
+ vhci->port_power_mask |= BIT(wIndex);
+ return status;
+ }
+ if (wValue == USB_PORT_FEAT_RESET) {
+ pr_debug("t2bce_vhci: hub SetPortFeature RESET port=%u\n", wIndex);
+ return bce_vhci_reset_device(vhci, wIndex, wValue);
+ }
+ if (wValue == USB_PORT_FEAT_SUSPEND) {
+ /* Trace the usbcore hub state machine against the firmware port commands. */
+ pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u\n", wIndex);
+ if (vhci->system_suspending) {
+ pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u skipped during system suspend\n",
+ wIndex);
+ return 0;
+ }
+ status = bce_vhci_cmd_port_suspend(&vhci->cq, (u8) wIndex);
+ pr_debug("t2bce_vhci: hub SetPortFeature SUSPEND port=%u -> %d\n", wIndex, status);
+ return status;
+ }
+ } else if (typeReq == ClearPortFeature) {
+ if (wValue == USB_PORT_FEAT_ENABLE)
+ return bce_vhci_cmd_port_disable(&vhci->cq, (u8) wIndex);
+ if (wValue == USB_PORT_FEAT_POWER) {
+ status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) wIndex);
+ if (!status)
+ vhci->port_power_mask &= ~BIT(wIndex);
+ return status;
+ }
+ if (wValue == USB_PORT_FEAT_C_CONNECTION) {
+ pr_debug("t2bce_vhci: hub ClearPortFeature C_CONNECTION port=%u\n", wIndex);
+ clear_bit(wIndex - 1, &vhci->port_change_pending);
+ return bce_vhci_cmd_port_status(&vhci->cq, (u8) wIndex, 0x40000, &port_status);
+ }
+ if (wValue == USB_PORT_FEAT_C_RESET) {
+ pr_debug("t2bce_vhci: hub ClearPortFeature C_RESET port=%u\n", wIndex);
+ return 0;
+ }
+ if (wValue == USB_PORT_FEAT_SUSPEND) {
+ pr_debug("t2bce_vhci: hub ClearPortFeature SUSPEND port=%u\n", wIndex);
+ status = bce_vhci_cmd_port_resume(&vhci->cq, (u8) wIndex);
+ pr_debug("t2bce_vhci: hub ClearPortFeature SUSPEND port=%u -> %d\n", wIndex, status);
+ return status;
+ }
+ }
+ pr_err("t2bce_vhci: bce_vhci_hub_control unhandled request: %x %i %i [bufl=%i]\n", typeReq, wValue, wIndex, wLength);
+ dump_stack();
+ return -EIO;
+}
+
+static int bce_vhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ struct bce_vhci_device *vdev;
+ bce_vhci_device_t devid;
+ pr_debug("t2bce_vhci: enable_device port=%u\n", udev->portnum);
+
+ if (vhci->port_to_device[udev->portnum])
+ return 0;
+
+ /* bridgeOS requires a firmware device id before endpoints are created. */
+ if (bce_vhci_cmd_device_create(&vhci->cq, udev->portnum, &devid))
+ return -EIO;
+
+ pr_debug("t2bce_vhci: device_create port=%u dev=%u\n", udev->portnum, devid);
+
+ vdev = kzalloc(sizeof(struct bce_vhci_device), GFP_KERNEL);
+ vhci->port_to_device[udev->portnum] = devid;
+ vhci->devices[devid] = vdev;
+
+ bce_vhci_create_transfer_queue(vhci, &vdev->tq[0], &udev->ep0, devid, DMA_BIDIRECTIONAL);
+ udev->ep0.hcpriv = &vdev->tq[0];
+ vdev->tq_mask |= BIT(0);
+
+ bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &udev->ep0.desc);
+ return 0;
+}
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,8,0)
+static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
+#else
+static int bce_vhci_address_device(struct usb_hcd *hcd, struct usb_device *udev, unsigned int timeout_ms)
+#endif
+{
+ /*
+ * Older kernels create the firmware device from enable_device(); newer
+ * kernels use address_device() for the same bridgeOS operation. The
+ * firmware command path does not currently consume the usbcore timeout.
+ */
+ return bce_vhci_enable_device(hcd, udev);
+}
+
+static void bce_vhci_free_device(struct usb_hcd *hcd, struct usb_device *udev)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ int i;
+ bce_vhci_device_t devid;
+ struct bce_vhci_device *dev;
+ pr_debug("t2bce_vhci: free_device port=%u\n", udev->portnum);
+ if (!vhci->port_to_device[udev->portnum])
+ return;
+ devid = vhci->port_to_device[udev->portnum];
+ dev = vhci->devices[devid];
+ for (i = 0; i < 32; i++) {
+ if (dev->tq_mask & BIT(i)) {
+ bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN);
+ bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, dev->tq[i].endp_addr);
+ if (dev->tq[i].endp)
+ dev->tq[i].endp->hcpriv = NULL;
+ bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]);
+ }
+ }
+ vhci->devices[devid] = NULL;
+ vhci->port_to_device[udev->portnum] = 0;
+ bce_vhci_cmd_device_destroy(&vhci->cq, devid);
+ kfree(dev);
+}
+
+static int bce_vhci_reset_device(struct bce_vhci *vhci, int index, u16 timeout)
+{
+ struct bce_vhci_device *dev = NULL;
+ bce_vhci_device_t devid;
+ int i;
+ int status;
+ enum dma_data_direction dir;
+ pr_debug("t2bce_vhci: reset_device port=%d\n", index);
+
+ devid = vhci->port_to_device[index];
+ if (devid) {
+ dev = vhci->devices[devid];
+
+ for (i = 0; i < 32; i++) {
+ if (dev->tq_mask & BIT(i)) {
+ bce_vhci_transfer_queue_pause(&dev->tq[i], BCE_VHCI_PAUSE_SHUTDOWN);
+ bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, dev->tq[i].endp_addr);
+ if (dev->tq[i].endp)
+ dev->tq[i].endp->hcpriv = NULL;
+ bce_vhci_destroy_transfer_queue(vhci, &dev->tq[i]);
+ }
+ }
+ vhci->devices[devid] = NULL;
+ vhci->port_to_device[index] = 0;
+ bce_vhci_cmd_device_destroy(&vhci->cq, devid);
+ }
+ status = bce_vhci_cmd_port_reset(&vhci->cq, (u8) index, timeout);
+
+ if (dev) {
+ if ((status = bce_vhci_cmd_device_create(&vhci->cq, index, &devid)))
+ return status;
+ vhci->devices[devid] = dev;
+ vhci->port_to_device[index] = devid;
+
+ for (i = 0; i < 32; i++) {
+ if (dev->tq_mask & BIT(i)) {
+ dir = usb_endpoint_dir_in(&dev->tq[i].endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
+ if (i == 0)
+ dir = DMA_BIDIRECTIONAL;
+ bce_vhci_create_transfer_queue(vhci, &dev->tq[i], dev->tq[i].endp, devid, dir);
+ dev->tq[i].endp->hcpriv = &dev->tq[i];
+ bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &dev->tq[i].endp->desc);
+ }
+ }
+ }
+
+ return status;
+}
+
+static int bce_vhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
+{
+ return 0;
+}
+
+static int bce_vhci_get_frame_number(struct usb_hcd *hcd)
+{
+ return 0;
+}
+
+static int bce_vhci_suspend_quiesce(struct usb_hcd *hcd)
+{
+ int i, j;
+ int status;
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ struct bce_vhci_transfer_queue *tq;
+ struct urb *urb;
+ struct bce_vhci_urb *vurb;
+ unsigned long flags;
+
+ pr_debug("t2bce_vhci: suspend started\n");
+
+ pr_debug("t2bce_vhci: suspend endpoints\n");
+ for (i = 0; i < 16; i++) {
+ if (!vhci->port_to_device[i])
+ continue;
+ for (j = 0; j < 32; j++) {
+ if (!(vhci->devices[vhci->port_to_device[i]]->tq_mask & BIT(j)))
+ continue;
+ tq = &vhci->devices[vhci->port_to_device[i]]->tq[j];
+ if (tq->endp_addr == 0x00 || (usb_endpoint_xfer_int(&tq->endp->desc) &&
+ usb_endpoint_dir_in(&tq->endp->desc))) {
+ spin_lock_irqsave(&tq->urb_lock, flags);
+ if (!list_empty(&tq->endp->urb_list)) {
+ urb = list_first_entry(&tq->endp->urb_list, struct urb, urb_list);
+ vurb = urb->hcpriv;
+ pr_debug("t2bce_vhci: suspend tq pre: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=1 first_state=%x\n",
+ tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled,
+ vurb ? vurb->state : 0xffffffff);
+ } else {
+ pr_debug("t2bce_vhci: suspend tq pre: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=0\n",
+ tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled);
+ }
+ spin_unlock_irqrestore(&tq->urb_lock, flags);
+ }
+
+ bce_vhci_transfer_queue_suspend_pause(tq);
+
+ if (tq->endp_addr == 0x00 || (usb_endpoint_xfer_int(&tq->endp->desc) &&
+ usb_endpoint_dir_in(&tq->endp->desc))) {
+ spin_lock_irqsave(&tq->urb_lock, flags);
+ if (!list_empty(&tq->endp->urb_list)) {
+ urb = list_first_entry(&tq->endp->urb_list, struct urb, urb_list);
+ vurb = urb->hcpriv;
+ pr_debug("t2bce_vhci: suspend tq post: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=1 first_state=%x\n",
+ tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled,
+ vurb ? vurb->state : 0xffffffff);
+ } else {
+ pr_debug("t2bce_vhci: suspend tq post: dev=%u ep=%02x active=%u paused_by=%x stalled=%u has_urb=0\n",
+ tq->dev_addr, tq->endp_addr, tq->active, tq->paused_by, tq->stalled);
+ }
+ spin_unlock_irqrestore(&tq->urb_lock, flags);
+ }
+ }
+ }
+
+ /*
+ * Only occupied virtual ports are suspended. bridgeOS rejects holes in
+ * the virtual hub, and that rejection would abort system PM.
+ */
+ pr_debug("t2bce_vhci: suspend occupied ports\n");
+ for (i = 1; i < ARRAY_SIZE(vhci->port_to_device); i++) {
+ if (!vhci->port_to_device[i])
+ continue;
+
+ status = bce_vhci_cmd_port_suspend(&vhci->cq, (u8)i);
+ pr_debug("t2bce_vhci: suspend port %d -> %d\n", i, status);
+ if (status)
+ return status;
+ }
+
+ /* Keep Linux-side event queues quiet before the BCE state-save transition. */
+ bce_vhci_event_queue_pause(&vhci->ev_commands);
+ bce_vhci_event_queue_pause(&vhci->ev_system);
+ bce_vhci_event_queue_pause(&vhci->ev_isochronous);
+ bce_vhci_event_queue_pause(&vhci->ev_interrupt);
+ bce_vhci_event_queue_pause(&vhci->ev_asynchronous);
+ pr_debug("t2bce_vhci: suspend done\n");
+ return 0;
+}
+
+static int bce_vhci_suspend_prepare(struct usb_hcd *hcd)
+{
+ /*
+ * VHCI must quiesce usbcore-visible state before the core BCE mailbox
+ * suspend command asks bridgeOS to save or discard queue state.
+ */
+ return bce_vhci_suspend_quiesce(hcd);
+}
+
+static int bce_vhci_bus_suspend(struct usb_hcd *hcd)
+{
+ int status;
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+
+ pr_debug("t2bce_vhci: bus_suspend entry\n");
+ vhci->port_change_pending = 0;
+ vhci->system_suspending = true;
+ status = bce_vhci_suspend_prepare(hcd);
+ if (status)
+ vhci->system_suspending = false;
+ pr_info("t2bce_vhci: bus_suspend exit status=%d\n", status);
+ return status;
+}
+
+static int bce_vhci_resume_no_state(struct usb_hcd *hcd)
+{
+ int i;
+ int status;
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+
+ bce_vhci_event_queue_resume(&vhci->ev_system);
+ bce_vhci_event_queue_resume(&vhci->ev_isochronous);
+ bce_vhci_event_queue_resume(&vhci->ev_interrupt);
+ bce_vhci_event_queue_resume(&vhci->ev_asynchronous);
+ bce_vhci_event_queue_resume(&vhci->ev_commands);
+
+ pr_debug("t2bce_vhci: no-state resume, rebuilding controller state\n");
+ vhci->stateful_resume = false;
+ bce_vhci_forget_devices(vhci);
+ status = bce_vhci_start_controller(vhci);
+ if (status) {
+ pr_info("t2bce_vhci: no-state resume exit status=%d\n", status);
+ return status;
+ }
+
+ pr_debug("t2bce_vhci: no-state resume, powering off all ports\n");
+ for (i = 1; i <= vhci->port_count; i++) {
+ status = bce_vhci_cmd_port_power_off(&vhci->cq, (u8) i);
+ pr_debug("t2bce_vhci: no-state resume, port %d power_off -> %d\n", i, status);
+ }
+
+ msleep(100);
+
+ pr_debug("t2bce_vhci: no-state resume, powering on all ports\n");
+ for (i = 1; i <= vhci->port_count; i++) {
+ status = bce_vhci_cmd_port_power_on(&vhci->cq, (u8) i);
+ pr_debug("t2bce_vhci: no-state resume, port %d power_on -> %d\n", i, status);
+ }
+
+ pr_debug("t2bce_vhci: no-state resume, notifying usbcore about lost power\n");
+ usb_root_hub_lost_power(hcd->self.root_hub);
+ vhci->no_state_resume = false;
+ pr_info("t2bce_vhci: no-state resume exit status=%d\n", status);
+ return status;
+}
+
+static int bce_vhci_resume_all_queues(struct bce_vhci *vhci)
+{
+ int devid;
+ int endp;
+ int status;
+
+ for (devid = 1; devid < ARRAY_SIZE(vhci->devices); devid++) {
+ if (!vhci->devices[devid])
+ continue;
+
+ for (endp = 0; endp < 32; endp++) {
+ if (!(vhci->devices[devid]->tq_mask & BIT(endp)))
+ continue;
+
+ if (vhci->devices[devid]->tq[endp].endp_addr == 0x00)
+ pr_debug("t2bce_vhci: resume_all EP0 pre dev=%d paused_by=%x state=%x active=%u\n",
+ devid, vhci->devices[devid]->tq[endp].paused_by,
+ vhci->devices[devid]->tq[endp].state,
+ vhci->devices[devid]->tq[endp].active);
+ status = bce_vhci_transfer_queue_resume(&vhci->devices[devid]->tq[endp],
+ BCE_VHCI_PAUSE_SUSPEND);
+ if (vhci->devices[devid]->tq[endp].endp_addr == 0x00)
+ pr_debug("t2bce_vhci: resume_all EP0 post dev=%d status=%d paused_by=%x state=%x active=%u\n",
+ devid, status, vhci->devices[devid]->tq[endp].paused_by,
+ vhci->devices[devid]->tq[endp].state,
+ vhci->devices[devid]->tq[endp].active);
+ if (status) {
+ pr_err("t2bce_vhci: stateful resume queue failed: dev=%d ep=%02x status=%d\n",
+ devid, vhci->devices[devid]->tq[endp].endp_addr, status);
+ return status;
+ }
+ }
+ }
+
+ return 0;
+}
+
+static int bce_vhci_resume_stateful(struct usb_hcd *hcd)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ int status;
+
+ /* Linux preserved-state resume path. This is kept separate from the
+ * no-state rebuild path below. */
+ vhci->port_change_pending = 0;
+ vhci->stateful_suspended_bounce = 0;
+ vhci->stateful_resume = true;
+
+ bce_vhci_event_queue_resume(&vhci->ev_system);
+ bce_vhci_event_queue_resume(&vhci->ev_isochronous);
+ bce_vhci_event_queue_resume(&vhci->ev_interrupt);
+ bce_vhci_event_queue_resume(&vhci->ev_asynchronous);
+
+ pr_debug("t2bce_vhci: stateful resume queues before firmware command events\n");
+ status = bce_vhci_resume_all_queues(vhci);
+ pr_debug("t2bce_vhci: stateful resume enabling firmware command events status=%d\n", status);
+ bce_vhci_event_queue_resume(&vhci->ev_commands);
+ pr_info("t2bce_vhci: stateful resume exit status=%d\n", status);
+ return status;
+}
+
+static int bce_vhci_bus_resume(struct usb_hcd *hcd)
+{
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ int status;
+
+ pr_debug("t2bce_vhci: bus_resume entry no_state_resume=%d\n", vhci->no_state_resume);
+ vhci->port_change_pending = 0;
+ vhci->system_suspending = false;
+ pr_debug("t2bce_vhci: resume started\n");
+
+ if (vhci->no_state_resume)
+ status = bce_vhci_resume_no_state(hcd);
+ else
+ status = bce_vhci_resume_stateful(hcd);
+
+ pr_info("t2bce_vhci: bus_resume exit status=%d no_state_resume=%d\n", status, vhci->no_state_resume);
+ return status;
+}
+
+static int bce_vhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
+{
+ struct bce_vhci_transfer_queue *q = urb->ep->hcpriv;
+ if (!q)
+ return -ENOENT;
+ pr_debug("t2bce_vhci: urb_enqueue dev=%u ep=%02x\n",
+ q->dev_addr, urb->ep->desc.bEndpointAddress);
+ return bce_vhci_urb_create(q, urb);
+}
+
+static int bce_vhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
+{
+ struct bce_vhci_transfer_queue *q = urb->ep->hcpriv;
+ if (!q)
+ return -ENOENT;
+ pr_debug("t2bce_vhci: urb_dequeue ep=%02x\n", urb->ep->desc.bEndpointAddress);
+ return bce_vhci_urb_request_cancel(q, urb, status);
+}
+
+static void bce_vhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep)
+{
+ struct bce_vhci_transfer_queue *q = ep->hcpriv;
+ pr_debug("t2bce_vhci: endpoint_reset\n");
+ if (q)
+ bce_vhci_transfer_queue_request_reset(q);
+}
+
+static u8 bce_vhci_endpoint_index(u8 addr)
+{
+ if (addr & 0x80)
+ return (u8) (0x10 + (addr & 0xf));
+ return (u8) (addr & 0xf);
+}
+
+static int bce_vhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp)
+{
+ u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress);
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ bce_vhci_device_t devid = vhci->port_to_device[udev->portnum];
+ struct bce_vhci_device *vdev = vhci->devices[devid];
+ pr_debug("t2bce_vhci: add_endpoint port=%u dev=%u ep_index=%u\n",
+ udev->portnum, devid, endp_index);
+
+ if (udev->bus->root_hub == udev) /* The USB hub */
+ return 0;
+ if (vdev == NULL)
+ return -ENODEV;
+ if (vdev->tq_mask & BIT(endp_index)) {
+ endp->hcpriv = &vdev->tq[endp_index];
+ return 0;
+ }
+
+ bce_vhci_create_transfer_queue(vhci, &vdev->tq[endp_index], endp, devid,
+ usb_endpoint_dir_in(&endp->desc) ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
+ endp->hcpriv = &vdev->tq[endp_index];
+ vdev->tq_mask |= BIT(endp_index);
+
+ bce_vhci_cmd_endpoint_create(&vhci->cq, devid, &endp->desc);
+ return 0;
+}
+
+static int bce_vhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *endp)
+{
+ u8 endp_index = bce_vhci_endpoint_index(endp->desc.bEndpointAddress);
+ struct bce_vhci *vhci = bce_vhci_from_hcd(hcd);
+ bce_vhci_device_t devid = vhci->port_to_device[udev->portnum];
+ struct bce_vhci_transfer_queue *q = endp->hcpriv;
+ struct bce_vhci_device *vdev;
+ pr_debug("t2bce_vhci: drop_endpoint port=%u ep_index=%u\n", udev->portnum, endp_index);
+ if (!devid || !vhci->devices[devid]) {
+ endp->hcpriv = NULL;
+ return 0;
+ }
+ vdev = vhci->devices[devid];
+ if (!q) {
+ if (vdev->tq_mask & BIT(endp_index)) {
+ pr_err("t2bce_vhci: endpoint missing hcpriv while still tracked\n");
+ q = &vdev->tq[endp_index];
+ } else {
+ return 0;
+ }
+ }
+
+ bce_vhci_cmd_endpoint_destroy(&vhci->cq, devid, (u8) (endp->desc.bEndpointAddress & 0x8Fu));
+ vdev->tq_mask &= ~BIT(endp_index);
+ bce_vhci_destroy_transfer_queue(vhci, q);
+ endp->hcpriv = NULL;
+ return 0;
+}
+
+static int bce_vhci_create_message_queues(struct bce_vhci *vhci)
+{
+ if (bce_vhci_message_queue_create(vhci, &vhci->msg_commands, "VHC1HostCommands") ||
+ bce_vhci_message_queue_create(vhci, &vhci->msg_system, "VHC1HostSystemEvents") ||
+ bce_vhci_message_queue_create(vhci, &vhci->msg_isochronous, "VHC1HostIsochronousEvents") ||
+ bce_vhci_message_queue_create(vhci, &vhci->msg_interrupt, "VHC1HostInterruptEvents") ||
+ bce_vhci_message_queue_create(vhci, &vhci->msg_asynchronous, "VHC1HostAsynchronousEvents")) {
+ bce_vhci_destroy_message_queues(vhci);
+ return -EINVAL;
+ }
+ spin_lock_init(&vhci->msg_asynchronous_lock);
+ bce_vhci_command_queue_create(&vhci->cq, &vhci->msg_commands);
+ return 0;
+}
+
+static void bce_vhci_destroy_message_queues(struct bce_vhci *vhci)
+{
+ bce_vhci_command_queue_destroy(&vhci->cq);
+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_commands);
+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_system);
+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_isochronous);
+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_interrupt);
+ bce_vhci_message_queue_destroy(vhci, &vhci->msg_asynchronous);
+}
+
+static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg);
+static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg);
+
+static int bce_vhci_create_event_queues(struct bce_vhci *vhci)
+{
+ vhci->ev_cq = t2bce_core_create_cq(vhci->client, 0x100);
+ if (!vhci->ev_cq)
+ return -EINVAL;
+#define CREATE_EVENT_QUEUE(field, name, cb) bce_vhci_event_queue_create(vhci, &vhci->field, name, cb)
+ if (__bce_vhci_event_queue_create(vhci, &vhci->ev_commands, "VHC1FirmwareCommands",
+ bce_vhci_firmware_event_completion) ||
+ CREATE_EVENT_QUEUE(ev_system, "VHC1FirmwareSystemEvents", bce_vhci_handle_system_event) ||
+ CREATE_EVENT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents", bce_vhci_handle_usb_event) ||
+ CREATE_EVENT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents", bce_vhci_handle_usb_event) ||
+ CREATE_EVENT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents", bce_vhci_handle_usb_event)) {
+ bce_vhci_destroy_event_queues(vhci);
+ return -EINVAL;
+ }
+#undef CREATE_EVENT_QUEUE
+ return 0;
+}
+
+static void bce_vhci_destroy_event_queues(struct bce_vhci *vhci)
+{
+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_commands);
+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_system);
+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_isochronous);
+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_interrupt);
+ bce_vhci_event_queue_destroy(vhci, &vhci->ev_asynchronous);
+ if (vhci->ev_cq)
+ t2bce_core_destroy_cq(vhci->client, vhci->ev_cq);
+}
+
+static void bce_vhci_send_fw_event_response(struct bce_vhci *vhci, struct bce_vhci_message *req, u16 status)
+{
+ unsigned long timeout = 1000;
+ struct bce_vhci_message r = *req;
+ r.cmd = (u16) (req->cmd | 0x8000u);
+ r.status = status;
+ r.param1 = req->param1;
+ r.param2 = 0;
+
+ if (t2bce_core_reserve_submission(vhci->msg_system.sq, &timeout)) {
+ pr_err("t2bce_vhci: Cannot reserve submision for FW event reply\n");
+ return;
+ }
+ bce_vhci_message_queue_write(&vhci->msg_system, &r);
+}
+
+static int bce_vhci_handle_firmware_event(struct bce_vhci *vhci, struct bce_vhci_message *msg)
+{
+ unsigned long flags;
+ bce_vhci_device_t devid;
+ u8 endp;
+ struct bce_vhci_device *dev;
+ struct bce_vhci_transfer_queue *tq;
+ if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE || msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) {
+ devid = (bce_vhci_device_t) (msg->param1 & 0xff);
+ endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff));
+ dev = vhci->devices[devid];
+ if (!dev || !(dev->tq_mask & BIT(endp)))
+ return BCE_VHCI_BAD_ARGUMENT;
+ tq = &dev->tq[endp];
+ }
+
+ if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_REQUEST_STATE) {
+ if (msg->param2 == BCE_VHCI_ENDPOINT_ACTIVE) {
+ int ret = bce_vhci_transfer_queue_resume(tq, BCE_VHCI_PAUSE_FIRMWARE);
+
+ pr_debug("t2bce_vhci: FW endpoint request ACTIVE dev=%u ep=%02x paused_by=%x state=%x active=%u ret=%d reply=%u\n",
+ devid, tq->endp_addr, tq->paused_by, tq->state, tq->active, ret,
+ ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS);
+ return ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS;
+ } else if (msg->param2 == BCE_VHCI_ENDPOINT_PAUSED) {
+ int ret = bce_vhci_transfer_queue_pause(tq, BCE_VHCI_PAUSE_FIRMWARE);
+
+ pr_debug("t2bce_vhci: FW endpoint request PAUSED dev=%u ep=%02x paused_by=%x state=%x active=%u ret=%d reply=%u\n",
+ devid, tq->endp_addr, tq->paused_by, tq->state, tq->active, ret,
+ ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS);
+ return ret ? BCE_VHCI_ERROR : BCE_VHCI_SUCCESS;
+ }
+ return BCE_VHCI_BAD_ARGUMENT;
+ } else if (msg->cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE) {
+ if (msg->param2 == BCE_VHCI_ENDPOINT_STALLED) {
+ tq->state = msg->param2;
+ spin_lock_irqsave(&tq->urb_lock, flags);
+ tq->active = false;
+ tq->stalled = true;
+ spin_unlock_irqrestore(&tq->urb_lock, flags);
+ pr_warn("t2bce_vhci: firmware endpoint stalled dev=%u ep=%02x paused_by=%x; %s\n",
+ devid, tq->endp_addr, tq->paused_by,
+ tq->endp_addr == 0x00 ? "scheduling EP0 reset" : "leaving stalled");
+ if (tq->endp_addr == 0x00)
+ bce_vhci_transfer_queue_request_reset(tq);
+ return BCE_VHCI_SUCCESS;
+ }
+ return BCE_VHCI_BAD_ARGUMENT;
+ }
+ pr_warn("t2bce_vhci: Unhandled firmware event: %x s=%x p1=%x p2=%llx\n",
+ msg->cmd, msg->status, msg->param1, msg->param2);
+ return BCE_VHCI_BAD_ARGUMENT;
+}
+
+static void bce_vhci_handle_firmware_events_w(struct work_struct *ws)
+{
+ size_t cnt = 0;
+ int result;
+ struct bce_vhci *vhci = container_of(ws, struct bce_vhci, w_fw_events);
+ struct t2bce_core_queue_sq *sq = vhci->ev_commands.sq;
+ struct t2bce_core_sq_completion_data *cq;
+ struct bce_vhci_message *msg, *msg2 = NULL;
+
+ while (true) {
+ if (msg2) {
+ msg = msg2;
+ msg2 = NULL;
+ } else if ((cq = t2bce_core_next_completion(sq))) {
+ if (cq->status == T2BCE_COMPLETION_ABORTED) {
+ t2bce_core_notify_submission_complete(sq);
+ continue;
+ }
+ msg = &vhci->ev_commands.data[t2bce_core_queue_sq_head(sq)];
+ } else {
+ break;
+ }
+
+ pr_debug("t2bce_vhci: Got fw event: %x s=%x p1=%x p2=%llx\n", msg->cmd, msg->status, msg->param1, msg->param2);
+ if ((cq = t2bce_core_next_completion(sq))) {
+ msg2 = &vhci->ev_commands.data[(t2bce_core_queue_sq_head(sq) + 1) % t2bce_core_queue_sq_capacity(sq)];
+ pr_debug("t2bce_vhci: Got second fw event: %x s=%x p1=%x p2=%llx\n",
+ msg->cmd, msg->status, msg->param1, msg->param2);
+ if (cq->status != T2BCE_COMPLETION_ABORTED &&
+ msg2->cmd == (msg->cmd | 0x4000) && msg2->param1 == msg->param1) {
+ /* Take two elements */
+ pr_debug("t2bce_vhci: Cancelled\n");
+ bce_vhci_send_fw_event_response(vhci, msg, BCE_VHCI_ABORT);
+
+ t2bce_core_notify_submission_complete(sq);
+ t2bce_core_notify_submission_complete(sq);
+ msg2 = NULL;
+ cnt += 2;
+ continue;
+ }
+
+ pr_warn("t2bce_vhci: Handle fw event - unexpected cancellation\n");
+ }
+
+ result = bce_vhci_handle_firmware_event(vhci, msg);
+ bce_vhci_send_fw_event_response(vhci, msg, (u16) result);
+
+
+ t2bce_core_notify_submission_complete(sq);
+ ++cnt;
+ }
+ bce_vhci_event_queue_submit_pending(&vhci->ev_commands, cnt);
+ if (t2bce_core_queue_sq_available(sq) == t2bce_core_queue_sq_capacity(sq) - 1) {
+ pr_debug("t2bce_vhci: complete\n");
+ complete(&vhci->ev_commands.queue_empty_completion);
+ }
+}
+
+static void bce_vhci_firmware_event_completion(struct t2bce_core_queue_sq *sq)
+{
+ struct bce_vhci_event_queue *q = t2bce_core_queue_sq_userdata(sq);
+ queue_work(q->vhci->tq_state_wq, &q->vhci->w_fw_events);
+}
+
+static const char *bce_vhci_port_state_change_label(u64 p2)
+{
+ return (p2 & 0x4) ? "connected" : "disconnected";
+}
+
+static void bce_vhci_handle_system_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg)
+{
+ if (msg->cmd & 0x8000) {
+ bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg);
+ } else if (msg->cmd == BCE_VHCI_EV_PORT_STATUS_CHANGE &&
+ msg->param1 > 0 &&
+ msg->param1 < ARRAY_SIZE(q->vhci->port_to_device)) {
+ u32 normalized_status = (u32)msg->param2 & ~BCE_VHCI_PORT_STATUS_C_CONNECTION;
+
+ if (q->vhci->stateful_resume &&
+ (msg->param2 & BCE_VHCI_PORT_STATUS_CONNECTED) &&
+ normalized_status == 0x285) {
+ set_bit(msg->param1 - 1, &q->vhci->stateful_suspended_bounce);
+ pr_debug("t2bce_vhci: marked stateful suspended bounce port=%u raw=%llx normalized=%x\n",
+ msg->param1, msg->param2, normalized_status);
+ }
+ set_bit(msg->param1 - 1, &q->vhci->port_change_pending);
+ pr_debug("t2bce_vhci: port state change event: port=%u state=%s status=%x p2=%llx\n",
+ msg->param1, bce_vhci_port_state_change_label(msg->param2),
+ msg->status, msg->param2);
+ } else {
+ pr_warn("t2bce_vhci: Unhandled system event: %x s=%x p1=%x p2=%llx\n",
+ msg->cmd, msg->status, msg->param1, msg->param2);
+ }
+}
+
+static void bce_vhci_handle_usb_event(struct bce_vhci_event_queue *q, struct bce_vhci_message *msg)
+{
+ bce_vhci_device_t devid;
+ u8 endp;
+ struct bce_vhci_device *dev;
+ if (msg->cmd & 0x8000) {
+ bce_vhci_command_queue_deliver_completion(&q->vhci->cq, msg);
+ } else if (msg->cmd == BCE_VHCI_CMD_TRANSFER_REQUEST || msg->cmd == BCE_VHCI_CMD_CONTROL_TRANSFER_STATUS) {
+ devid = (bce_vhci_device_t) (msg->param1 & 0xff);
+ endp = bce_vhci_endpoint_index((u8) ((msg->param1 >> 8) & 0xff));
+ dev = q->vhci->devices[devid];
+ if (!dev || (dev->tq_mask & BIT(endp)) == 0) {
+ pr_err("t2bce_vhci: Didn't find destination for transfer queue event\n");
+ return;
+ }
+ bce_vhci_transfer_queue_event(&dev->tq[endp], msg);
+ } else {
+ pr_warn("t2bce_vhci: Unhandled USB event: %x s=%x p1=%x p2=%llx\n",
+ msg->cmd, msg->status, msg->param1, msg->param2);
+ }
+}
+
+
+
+static const struct hc_driver bce_vhci_driver = {
+ .description = "t2bce_vhci",
+ .product_desc = "BCE VHCI Host Controller",
+ .hcd_priv_size = sizeof(struct bce_vhci *),
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(5,4,0)
+ .flags = HCD_USB2,
+#else
+ .flags = HCD_USB2 | HCD_DMA,
+#endif
+
+ .start = bce_vhci_start,
+ .stop = bce_vhci_stop,
+ .hub_status_data = bce_vhci_hub_status_data,
+ .hub_control = bce_vhci_hub_control,
+ .urb_enqueue = bce_vhci_urb_enqueue,
+ .urb_dequeue = bce_vhci_urb_dequeue,
+ .enable_device = bce_vhci_enable_device,
+ .free_dev = bce_vhci_free_device,
+ .address_device = bce_vhci_address_device,
+ .add_endpoint = bce_vhci_add_endpoint,
+ .drop_endpoint = bce_vhci_drop_endpoint,
+ .endpoint_reset = bce_vhci_endpoint_reset,
+ .check_bandwidth = bce_vhci_check_bandwidth,
+ .get_frame_number = bce_vhci_get_frame_number,
+ .bus_suspend = bce_vhci_bus_suspend,
+ .bus_resume = bce_vhci_bus_resume
+};
+
+
+int __init bce_vhci_module_init(void)
+{
+ struct device *parent;
+ int result;
+
+ if ((result = alloc_chrdev_region(&bce_vhci_chrdev, 0, 1, "t2bce_vhci")))
+ goto fail_chrdev;
+#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0)
+ bce_vhci_class = class_create(THIS_MODULE, "t2bce_vhci");
+#else
+ bce_vhci_class = class_create("t2bce_vhci");
+#endif
+ if (IS_ERR(bce_vhci_class)) {
+ result = PTR_ERR(bce_vhci_class);
+ goto fail_class;
+ }
+
+ global_vhci = kzalloc(sizeof(*global_vhci), GFP_KERNEL);
+ if (!global_vhci) {
+ result = -ENOMEM;
+ goto fail_alloc;
+ }
+
+ parent = t2bce_core_device_get();
+ if (IS_ERR(parent)) {
+ result = PTR_ERR(parent);
+ goto fail_create;
+ }
+
+ result = bce_vhci_create(parent, global_vhci);
+ t2bce_core_device_put(parent);
+ if (result)
+ goto fail_create;
+
+ pr_info("t2bce_vhci: module initialized\n");
+ return 0;
+
+fail_create:
+ kfree(global_vhci);
+ global_vhci = NULL;
+fail_alloc:
+ class_destroy(bce_vhci_class);
+fail_class:
+ unregister_chrdev_region(bce_vhci_chrdev, 1);
+fail_chrdev:
+ if (!result)
+ result = -EINVAL;
+ pr_info("t2bce_vhci: module init failed status=%d\n", result);
+ return result;
+}
+void __exit bce_vhci_module_exit(void)
+{
+ if (global_vhci) {
+ bce_vhci_destroy(global_vhci);
+ kfree(global_vhci);
+ global_vhci = NULL;
+ }
+ class_destroy(bce_vhci_class);
+ unregister_chrdev_region(bce_vhci_chrdev, 1);
+ pr_info("t2bce_vhci: module exited\n");
+}
+
+module_param_named(vhci_port_mask, bce_vhci_port_mask, ushort, 0444);
+MODULE_PARM_DESC(vhci_port_mask, "Specifies which VHCI ports are enabled");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("André Eikmeyer <andre.eikmeyer@gmail.com>");
+MODULE_DESCRIPTION("Apple T2 BCE VHCI Driver");
+MODULE_VERSION("0.02");
+MODULE_SOFTDEP("pre: t2bce_core");
+module_init(bce_vhci_module_init);
+module_exit(bce_vhci_module_exit);
diff --git a/drivers/staging/t2bce/t2bce_vhci/vhci.h b/drivers/staging/t2bce/t2bce_vhci/vhci.h
new file mode 100644
index 000000000000..111111111111
--- /dev/null
+++ b/drivers/staging/t2bce/t2bce_vhci/vhci.h
@@ -0,0 +1,69 @@
+#ifndef BCE_VHCI_H
+#define BCE_VHCI_H
+
+#include "queue.h"
+#include "transfer.h"
+
+struct device;
+struct usb_hcd;
+struct t2bce_core_client;
+
+struct bce_vhci_device {
+ struct bce_vhci_transfer_queue tq[32];
+ u32 tq_mask;
+};
+struct bce_vhci {
+ struct t2bce_core_client *client;
+ dev_t vdevt;
+ struct device *vdev;
+ struct usb_hcd *hcd;
+ struct spinlock hcd_spinlock;
+ struct bce_vhci_message_queue msg_commands;
+ struct bce_vhci_message_queue msg_system;
+ struct bce_vhci_message_queue msg_isochronous;
+ struct bce_vhci_message_queue msg_interrupt;
+ struct bce_vhci_message_queue msg_asynchronous;
+ struct spinlock msg_asynchronous_lock;
+ struct bce_vhci_command_queue cq;
+ struct t2bce_core_queue_cq *ev_cq;
+ struct bce_vhci_event_queue ev_commands;
+ struct bce_vhci_event_queue ev_system;
+ struct bce_vhci_event_queue ev_isochronous;
+ struct bce_vhci_event_queue ev_interrupt;
+ struct bce_vhci_event_queue ev_asynchronous;
+ u16 port_mask;
+ u8 port_count;
+ u16 port_power_mask;
+ bce_vhci_device_t port_to_device[17];
+ struct bce_vhci_device *devices[17];
+ struct workqueue_struct *tq_state_wq;
+ struct work_struct w_fw_events;
+ struct work_struct w_add_hcd;
+ unsigned long port_change_pending;
+ unsigned long stateful_suspended_bounce;
+ bool stateful_resume;
+ bool no_state_resume;
+ bool hcd_registered;
+ bool system_suspending;
+};
+
+int __init bce_vhci_module_init(void);
+void __exit bce_vhci_module_exit(void);
+
+int bce_vhci_create(struct device *parent, struct bce_vhci *vhci);
+void bce_vhci_destroy(struct bce_vhci *vhci);
+int bce_vhci_start(struct usb_hcd *hcd);
+void bce_vhci_stop(struct usb_hcd *hcd);
+int bce_vhci_add_hcd(struct bce_vhci *vhci);
+void bce_vhci_remove_hcd(struct bce_vhci *vhci);
+void bce_vhci_shutdown(struct bce_vhci *vhci);
+void bce_vhci_pm_reset(struct bce_vhci *vhci);
+void bce_vhci_pm_prepare(struct bce_vhci *vhci);
+void bce_vhci_pm_prepare_no_state(struct bce_vhci *vhci);
+void bce_vhci_pm_mark_no_state_resume(struct bce_vhci *vhci);
+bool bce_vhci_pm_is_no_state_resume(struct bce_vhci *vhci);
+void bce_vhci_pm_complete(struct bce_vhci *vhci);
+
+struct bce_vhci *bce_vhci_from_hcd(struct usb_hcd *hcd);
+
+#endif //BCE_VHCI_H
--
Armbian
Reference in New Issue View Git Blame Copy Permalink