first and last commit

drivers/apple/csi/include/csi_private.h

@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2011-2013 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#ifndef __CSI_PRIVATE_H
+#define __CSI_PRIVATE_H
+
+#include <drivers/csi.h>
+#include <debug.h>
+
+// NOTE:  The following header file is expected to be found via the search paths expressed in
+// rules.mk via the GLOBAL_INCLUDES macro.
+#include <csi_prot_common.h>
+#include <csi_platform_defines.h>
+#include <csi_system.h>
+
+// build-time CSI protocol version range check
+// Make sure you could this in sync with the protoc
+#define EXPECTED_RANGE_MIN_PROTOCOL_VER_MAJOR 9
+#define EXPECTED_RANGE_MAX_PROTOCOL_VER_MAJOR 9
+
+#if !((EXPECTED_RANGE_MIN_PROTOCOL_VER_MAJOR <= APPLE_CSI_PROTOCOL_VER_MAJOR) && (EXPECTED_RANGE_MAX_PROTOCOL_VER_MAJOR >= APPLE_CSI_PROTOCOL_VER_MAJOR))
+#error ERROR: This iBoot is not compatible with your AppleStorageProcessor firmware
+#endif
+
+
+// explicitly specify task stack sizes
+// empirically derived stack high watermark measurements
+// csi inbox_not_e:  512
+// csi ep mgmt:      416
+// csi ep crashlog:  1728
+// csi ep console:   448
+// csi ep syslog:    1424
+
+// request 8 kb to cover with a healthy margin
+// further tuning will be handled with <rdar://problem/15208515> Analyze stack usage and tune stack size for csi iBoot tasks
+
+#define kTaskStackSize_inbox_not_empty  0x2000
+#define kTaskStackSize_endpoints        0x2000
+
+
+
+
+// default send message time out
+#define CSI_SEND_MESSAGE_TIMEOUT  (10 * 1000 * 1000) // 10s
+
+
+#define roundup(x,y)        ((((x)+(y)-1)/(y))*(y))
+
+#define is_power_of_2(x)    (((x)==((x)&~((x)-1)))&&((x)!=0))
+
+
+#ifndef REQUIRE
+#define REQUIRE(_expr)                                      \
+            do {                                            \
+                if (__builtin_expect(!(_expr), 0))          \
+                    panic("%s:%s:%u: REQUIRE failed: %s\n", \
+                        __FILE__,                           \
+                        __PRETTY_FUNCTION__,                \
+                        __LINE__, #_expr);                  \
+            } while(0);
+#endif
+
+
+#define COPROC_STATE(s) \
+  (((s)==MGMT_INITIALIZED) ?                          "Initialized" : \
+    (((s)==MGMT_QUIESCED) ?                           "Quiesced" : \
+      (((s)==MGMT_WAITING_FOR_ACK) ?                  "Waiting For Hello" : \
+        (((s)==MGMT_VERSION_OK) ?                     "Running" : \
+          (((s)==MGMT_UNSUPPORTED_PROTOCOL_VERSION) ? "Running - Unsupported Protocol Version" : \
+            (((s)==MGMT_NONEXISTENT) ?                "Uninitialized" : "???"))))))
+
+
+
+  // CS_LOG levels are passed through to dprintf() and compared against DEBUG_LEVEL.
+  // <debug.h> defines { DEBUG_SILENT DEBUG_CRITICAL DEBUG_INFO DEBUG_SPEW }
+  // one may use DEBUG_LEVEL to enable output and (DEBUG_LEVEL+1) to stifle output
+
+  // these levels should probably be enabled by default
+#define CSI_LOG_LEVEL_CRITICAL    DEBUG_CRITICAL    // indication of a serious problem
+
+  // these levels *may* be enabled by default
+#define CSI_LOG_LEVEL_CRASHLOG    DEBUG_CRITICAL    // coproc crashlog output
+#define CSI_LOG_LEVEL_CONSOLE     DEBUG_INFO        // coproc console output
+#define CSI_LOG_LEVEL_SYSLOG      DEBUG_CRITICAL    // coproc syslog output
+#define CSI_LOG_LEVEL_DEBUG_SPEW  DEBUG_SPEW        // verbose debug output
+
+  // these levels are disabled by default
+#if DEBUG_BUILD
+#define CSI_LOG_LEVEL_DEBUG       DEBUG_INFO
+#else
+#define CSI_LOG_LEVEL_DEBUG       DEBUG_CRITICAL        // every little operational detail - TMI - useful for debugging only
+#endif
+
+
+  // csi log formatting macros
+#define CSI_LOG_RAW(lev, x...) \
+  do { dprintf(CSI_LOG_LEVEL_##lev, x); } while(0)
+
+#define CSI_LOG(lev, cop, x...) \
+  do { dprintf(CSI_LOG_LEVEL_##lev, "%s: ", IOP_NAME(cop)); dprintf(CSI_LOG_LEVEL_##lev, x); } while(0)
+
+#define CSI_EP_LOG(lev, cop, ep_name, x...) \
+  do { dprintf(CSI_LOG_LEVEL_##lev, "[%s %s] ", IOP_NAME(cop), ep_name); dprintf(CSI_LOG_LEVEL_##lev, x); } while(0)
+
+
+  // firmware api
+
+typedef struct {
+  uint32_t                  alignment_req;
+  void                      *fw_address;
+  uint32_t                  fw_size;
+  void                      *heap_address;
+  uint32_t                  heap_size;
+  addr_t                    region_base;
+  uint32_t                  region_size;
+} fw_desc_t;
+
+csi_status_t  fw_fixup_and_relocate (csi_coproc_t which_coproc, fw_desc_t *fw_desc);
+void          fw_print_version (csi_coproc_t coproc, fw_desc_t *fw_desc);
+
+csi_status_t  csi_start (csi_coproc_t which_coproc);
+csi_status_t  csi_stop  (csi_coproc_t which_coproc);
+csi_status_t  csi_panic_recover (csi_coproc_t which_coproc);
+csi_status_t  quiesce_iop   (csi_coproc_t which_coproc);
+csi_status_t  hibernate_iop (csi_coproc_t which_coproc);
+csi_status_t  request_iop_power_state (csi_coproc_t which_coproc, csi_power_state_t ps);
+csi_status_t  set_send_timeout (csi_coproc_t which_coproc, uint32_t timeout);
+uint32_t      get_send_timeout (csi_coproc_t which_coproc);
+void          csi_info (csi_coproc_t which_coproc);
+
+
+csi_status_t  csi_set_feature (csi_coproc_t which_coproc, coproc_feature_t feature);
+csi_status_t  csi_clr_feature (csi_coproc_t which_coproc, coproc_feature_t feature);
+
+void          maybe_do_cache_operation(int operation, void *address, u_int32_t length);
+
+
+//
+// built-in endpoint api
+//
+
+csi_status_t  bi_ep_send_message (void *token, msg_payload_t msg_payload);
+
+#endif /* !defined(__CSI_PRIVATE_H) */

drivers/apple/csi/include/endpoints/console_ep.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2011-2013 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#pragma once
+
+//
+// Console object
+//
+
+typedef struct {
+  csi_coproc_t         coproc;
+  struct task_event    msg_event;
+  void                 *csi_token;
+  const char           *name;
+  uint32_t             verbosity;
+  void                 *buffers;
+  uint32_t             allocated;
+  char                 *scratch;
+} ep_console_desc_t;
+
+
+//
+// Console APIs
+//
+
+void          bi_ep_console_create          (csi_coproc_t which_coproc);
+csi_status_t  bi_ep_console_send_custom_cmd (csi_coproc_t which_coproc, uint32_t payload);
+csi_status_t  bi_ep_console_set_verbosity   (csi_coproc_t which_coproc, uint32_t verbosity);
+
+csi_status_t  console_quiesce (csi_coproc_t which_coproc, csi_power_state_t ps);

drivers/apple/csi/include/endpoints/crashlog_ep.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2011-2013 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#pragma once
+
+// crash log type
+typedef enum {
+  CSI_CRASHLOG_TYPE_NMI = 0,
+  CSI_CRASHLOG_TYPE_MSG_CRASH,
+  CSI_CRASHLOG_TYPE_MSG_LOG,
+} csi_crashlog_t;
+
+//
+// Crashlog Object
+//
+typedef struct {
+  csi_coproc_t        coproc;
+  struct task_event   msg_event;
+  void                *csi_token;
+  const char          *name;
+  void                *buffer;
+  uint32_t            allocated;
+} ep_crashlog_desc_t;
+
+
+//
+// Public API
+//
+
+void          bi_ep_crashlog_create      (csi_coproc_t which_coproc);
+csi_status_t  bi_ep_crashlog_force_event (csi_coproc_t which_coproc, csi_crashlog_t crashlog_type);
+
+csi_status_t  crashlog_quiesce (csi_coproc_t which_coproc, csi_power_state_t ps);

drivers/apple/csi/include/endpoints/management_ep.h

@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2011-2013 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#pragma once
+
+#include <csi_ipc_protocol.h>
+#include <csi_mgmt_protocol.h>
+
+
+// FixMe - auto-pinger not implemented
+// pinger
+typedef enum {
+  PINGER_DISABLED = 0x0000,
+  PINGER_OK       = 0x0001,
+  PINGER_TIMEOUT  = 0x1000,
+  PINGER_SEQ_ERR  = 0x1001,
+  PINGER_XMIT_ERR = 0x1002,
+} pinger_status_t;
+
+typedef struct {
+  uint32_t        xmt_count;
+  uint32_t        replied_to_count;
+  uint32_t        xmt_err_count;
+  uint32_t        rcv_err_count;
+  uint32_t        timeout_err_count;
+} ping_stats_t;
+
+typedef struct {
+  uint32_t           ping_period_usec;
+  uint32_t           report_period_usec;
+  pinger_status_t    status;
+
+  csi_ping_seqnum_t  xmt_seq_number;
+  csi_ping_seqnum_t  expected_rcv_seq_number;
+  csi_ping_seqnum_t  rcv_seq_number;
+
+  ping_stats_t       stats;
+} pinger_t;
+
+
+//
+// Management Object
+//
+
+typedef enum {
+  MGMT_INITIALIZED = 0,
+  MGMT_QUIESCED,
+  MGMT_WAITING_FOR_ACK,
+  MGMT_VERSION_OK,
+
+  // error status
+  MGMT_NONEXISTENT                  = 0x1000,
+  MGMT_UNSUPPORTED_PROTOCOL_VERSION = 0x1001,
+  MGMT_MBOX_ERROR                   = 0x1002,
+} mgmt_state_t;
+
+typedef struct {
+  csi_coproc_t                coproc;
+  struct task_event           msg_event;
+  void                        *csi_token;
+  const char                  *name;
+
+  csi_protocol_version_t      remoteVersion;
+  mgmt_state_t                state;
+
+  // pinger
+  pinger_t                    pinger;
+
+  volatile bool               ping_test_waiting;
+  volatile csi_ping_seqnum_t  ping_test_seqnum;
+  volatile ipc_msg_t          ping_test_ack;
+
+  // power
+  struct task_event           *ps_event;
+  csi_power_state_t           power_state;
+} ep_mgmt_desc_t;
+
+extern ep_mgmt_desc_t *mgmt_descs[];
+
+
+//
+// Endpoint API
+//
+
+void         bi_ep_mgmt_create    (csi_coproc_t which_coproc, struct task_event *ps_event);
+csi_status_t bi_ep_mgmt_send_ps   (csi_coproc_t which_coproc, csi_power_state_t ps);
+csi_status_t bi_ep_mgmt_send_ping (csi_coproc_t which_coproc, int ping_count);
+mgmt_state_t bi_ep_mgmt_get_state (csi_coproc_t which_coproc);
+
+csi_power_state_t bi_ep_mgmt_get_power_state (csi_coproc_t which_coproc);
+
+// to support panic recovery
+csi_status_t handle_power_state_ack (ep_mgmt_desc_t *mgmt, ipc_msg_t msg_in);

drivers/apple/csi/include/endpoints/syslog_ep.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2011-2013 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#pragma once
+
+#if DEBUG_BUILD
+#define LOG_DEFAULT LOG_DEBUG
+#else
+#define LOG_DEFAULT LOG_WARNING
+#endif
+
+
+//
+// Endpoint APIs
+//
+
+void          bi_ep_syslog_create    (csi_coproc_t which_coproc);
+csi_status_t  bi_ep_syslog_verbosity (csi_coproc_t which_coproc, uint32_t verbosity);
+
+csi_status_t  syslog_quiesce (csi_coproc_t which_coproc, csi_power_state_t ps);

drivers/apple/csi/include/queue.h

@@ -0,0 +1,768 @@
+/*
+ * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
+ * 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
+ * limitations under the License.
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
+ */
+/*-
+ * Copyright (c) 1991, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	@(#)queue.h	8.5 (Berkeley) 8/20/94
+ */
+
+#ifndef _SYS_QUEUE_H_
+#define	_SYS_QUEUE_H_
+
+/*
+ * This file defines five types of data structures: singly-linked lists,
+ * singly-linked tail queues, lists, tail queues, and circular queues.
+ *
+ * A singly-linked list is headed by a single forward pointer. The elements
+ * are singly linked for minimum space and pointer manipulation overhead at
+ * the expense of O(n) removal for arbitrary elements. New elements can be
+ * added to the list after an existing element or at the head of the list.
+ * Elements being removed from the head of the list should use the explicit
+ * macro for this purpose for optimum efficiency. A singly-linked list may
+ * only be traversed in the forward direction.  Singly-linked lists are ideal
+ * for applications with large datasets and few or no removals or for
+ * implementing a LIFO queue.
+ *
+ * A singly-linked tail queue is headed by a pair of pointers, one to the
+ * head of the list and the other to the tail of the list. The elements are
+ * singly linked for minimum space and pointer manipulation overhead at the
+ * expense of O(n) removal for arbitrary elements. New elements can be added
+ * to the list after an existing element, at the head of the list, or at the
+ * end of the list. Elements being removed from the head of the tail queue
+ * should use the explicit macro for this purpose for optimum efficiency.
+ * A singly-linked tail queue may only be traversed in the forward direction.
+ * Singly-linked tail queues are ideal for applications with large datasets
+ * and few or no removals or for implementing a FIFO queue.
+ *
+ * A list is headed by a single forward pointer (or an array of forward
+ * pointers for a hash table header). The elements are doubly linked
+ * so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before
+ * or after an existing element or at the head of the list. A list
+ * may only be traversed in the forward direction.
+ *
+ * A tail queue is headed by a pair of pointers, one to the head of the
+ * list and the other to the tail of the list. The elements are doubly
+ * linked so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before or
+ * after an existing element, at the head of the list, or at the end of
+ * the list. A tail queue may be traversed in either direction.
+ *
+ * A circle queue is headed by a pair of pointers, one to the head of the
+ * list and the other to the tail of the list. The elements are doubly
+ * linked so that an arbitrary element can be removed without a need to
+ * traverse the list. New elements can be added to the list before or after
+ * an existing element, at the head of the list, or at the end of the list.
+ * A circle queue may be traversed in either direction, but has a more
+ * complex end of list detection.
+ * Note that circle queues are deprecated, because, as the removal log
+ * in FreeBSD states, "CIRCLEQs are a disgrace to everything Knuth taught
+ * us in Volume 1 Chapter 2. [...] Use TAILQ instead, it provides the same
+ * functionality." Code using them will continue to compile, but they
+ * are no longer documented on the man page.
+ *
+ * For details on the use of these macros, see the queue(3) manual page.
+ *
+ *
+ *				SLIST	LIST	STAILQ	TAILQ	CIRCLEQ
+ * _HEAD			+	+	+	+	+
+ * _HEAD_INITIALIZER		+	+	+	+	-
+ * _ENTRY			+	+	+	+	+
+ * _INIT			+	+	+	+	+
+ * _EMPTY			+	+	+	+	+
+ * _FIRST			+	+	+	+	+
+ * _NEXT			+	+	+	+	+
+ * _PREV			-	-	-	+	+
+ * _LAST			-	-	+	+	+
+ * _FOREACH			+	+	+	+	+
+ * _FOREACH_SAFE		+	+	+	+	-
+ * _FOREACH_REVERSE		-	-	-	+	-
+ * _FOREACH_REVERSE_SAFE	-	-	-	+	-
+ * _INSERT_HEAD			+	+	+	+	+
+ * _INSERT_BEFORE		-	+	-	+	+
+ * _INSERT_AFTER		+	+	+	+	+
+ * _INSERT_TAIL			-	-	+	+	+
+ * _CONCAT			-	-	+	+	-
+ * _REMOVE_AFTER		+	-	+	-	-
+ * _REMOVE_HEAD			+	-	+	-	-
+ * _REMOVE_HEAD_UNTIL		-	-	+	-	-
+ * _REMOVE			+	+	+	+	+
+ * _SWAP			-	+	+	+	-
+ *
+ */
+#ifdef QUEUE_MACRO_DEBUG
+/* Store the last 2 places the queue element or head was altered */
+struct qm_trace {
+	char * lastfile;
+	int lastline;
+	char * prevfile;
+	int prevline;
+};
+
+#define	TRACEBUF	struct qm_trace trace;
+#define	TRASHIT(x)	do {(x) = (void *)-1;} while (0)
+
+#define	QMD_TRACE_HEAD(head) do {					\
+	(head)->trace.prevline = (head)->trace.lastline;		\
+	(head)->trace.prevfile = (head)->trace.lastfile;		\
+	(head)->trace.lastline = __LINE__;				\
+	(head)->trace.lastfile = __FILE__;				\
+} while (0)
+
+#define	QMD_TRACE_ELEM(elem) do {					\
+	(elem)->trace.prevline = (elem)->trace.lastline;		\
+	(elem)->trace.prevfile = (elem)->trace.lastfile;		\
+	(elem)->trace.lastline = __LINE__;				\
+	(elem)->trace.lastfile = __FILE__;				\
+} while (0)
+
+#else
+#define	QMD_TRACE_ELEM(elem)
+#define	QMD_TRACE_HEAD(head)
+#define	TRACEBUF
+#define	TRASHIT(x)
+#endif	/* QUEUE_MACRO_DEBUG */
+
+/*
+ * Singly-linked List declarations.
+ */
+#define	SLIST_HEAD(name, type)						\
+struct name {								\
+	struct type *slh_first;	/* first element */			\
+}
+
+#define	SLIST_HEAD_INITIALIZER(head)					\
+	{ NULL }
+
+#define	SLIST_ENTRY(type)						\
+struct {								\
+	struct type *sle_next;	/* next element */			\
+}
+
+/*
+ * Singly-linked List functions.
+ */
+#define	SLIST_EMPTY(head)	((head)->slh_first == NULL)
+
+#define	SLIST_FIRST(head)	((head)->slh_first)
+
+#define	SLIST_FOREACH(var, head, field)					\
+	for ((var) = SLIST_FIRST((head));				\
+	    (var);							\
+	    (var) = SLIST_NEXT((var), field))
+
+#define	SLIST_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = SLIST_FIRST((head));				\
+	    (var) && ((tvar) = SLIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	SLIST_FOREACH_PREVPTR(var, varp, head, field)			\
+	for ((varp) = &SLIST_FIRST((head));				\
+	    ((var) = *(varp)) != NULL;					\
+	    (varp) = &SLIST_NEXT((var), field))
+
+#define	SLIST_INIT(head) do {						\
+	SLIST_FIRST((head)) = NULL;					\
+} while (0)
+
+#define	SLIST_INSERT_AFTER(slistelm, elm, field) do {			\
+	SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field);	\
+	SLIST_NEXT((slistelm), field) = (elm);				\
+} while (0)
+
+#define	SLIST_INSERT_HEAD(head, elm, field) do {			\
+	SLIST_NEXT((elm), field) = SLIST_FIRST((head));			\
+	SLIST_FIRST((head)) = (elm);					\
+} while (0)
+
+#define	SLIST_NEXT(elm, field)	((elm)->field.sle_next)
+
+#define	SLIST_REMOVE(head, elm, type, field) do {			\
+	if (SLIST_FIRST((head)) == (elm)) {				\
+		SLIST_REMOVE_HEAD((head), field);			\
+	}								\
+	else {								\
+		struct type *curelm = SLIST_FIRST((head));		\
+		while (SLIST_NEXT(curelm, field) != (elm))		\
+			curelm = SLIST_NEXT(curelm, field);		\
+		SLIST_REMOVE_AFTER(curelm, field);			\
+	}								\
+	TRASHIT((elm)->field.sle_next);					\
+} while (0)
+
+#define SLIST_REMOVE_AFTER(elm, field) do {				\
+	SLIST_NEXT(elm, field) =					\
+	    SLIST_NEXT(SLIST_NEXT(elm, field), field);			\
+} while (0)
+
+#define	SLIST_REMOVE_HEAD(head, field) do {				\
+	SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field);	\
+} while (0)
+
+/*
+ * Singly-linked Tail queue declarations.
+ */
+#define	STAILQ_HEAD(name, type)						\
+struct name {								\
+	struct type *stqh_first;/* first element */			\
+	struct type **stqh_last;/* addr of last next element */		\
+}
+
+#define	STAILQ_HEAD_INITIALIZER(head)					\
+	{ NULL, &(head).stqh_first }
+
+#define	STAILQ_ENTRY(type)						\
+struct {								\
+	struct type *stqe_next;	/* next element */			\
+}
+
+/*
+ * Singly-linked Tail queue functions.
+ */
+#define	STAILQ_CONCAT(head1, head2) do {				\
+	if (!STAILQ_EMPTY((head2))) {					\
+		*(head1)->stqh_last = (head2)->stqh_first;		\
+		(head1)->stqh_last = (head2)->stqh_last;		\
+		STAILQ_INIT((head2));					\
+	}								\
+} while (0)
+
+#define	STAILQ_EMPTY(head)	((head)->stqh_first == NULL)
+
+#define	STAILQ_FIRST(head)	((head)->stqh_first)
+
+#define	STAILQ_FOREACH(var, head, field)				\
+	for((var) = STAILQ_FIRST((head));				\
+	   (var);							\
+	   (var) = STAILQ_NEXT((var), field))
+
+
+#define	STAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = STAILQ_FIRST((head));				\
+	    (var) && ((tvar) = STAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	STAILQ_INIT(head) do {						\
+	STAILQ_FIRST((head)) = NULL;					\
+	(head)->stqh_last = &STAILQ_FIRST((head));			\
+} while (0)
+
+#define	STAILQ_INSERT_AFTER(head, tqelm, elm, field) do {		\
+	if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+	STAILQ_NEXT((tqelm), field) = (elm);				\
+} while (0)
+
+#define	STAILQ_INSERT_HEAD(head, elm, field) do {			\
+	if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL)	\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+	STAILQ_FIRST((head)) = (elm);					\
+} while (0)
+
+#define	STAILQ_INSERT_TAIL(head, elm, field) do {			\
+	STAILQ_NEXT((elm), field) = NULL;				\
+	*(head)->stqh_last = (elm);					\
+	(head)->stqh_last = &STAILQ_NEXT((elm), field);			\
+} while (0)
+
+#define	STAILQ_LAST(head, type, field)					\
+	(STAILQ_EMPTY((head)) ?						\
+		NULL :							\
+	        ((struct type *)(void *)				\
+		((char *)((head)->stqh_last) - __offsetof(struct type, field))))
+
+#define	STAILQ_NEXT(elm, field)	((elm)->field.stqe_next)
+
+#define	STAILQ_REMOVE(head, elm, type, field) do {			\
+	if (STAILQ_FIRST((head)) == (elm)) {				\
+		STAILQ_REMOVE_HEAD((head), field);			\
+	}								\
+	else {								\
+		struct type *curelm = STAILQ_FIRST((head));		\
+		while (STAILQ_NEXT(curelm, field) != (elm))		\
+			curelm = STAILQ_NEXT(curelm, field);		\
+		STAILQ_REMOVE_AFTER(head, curelm, field);		\
+	}								\
+	TRASHIT((elm)->field.stqe_next);				\
+} while (0)
+
+#define	STAILQ_REMOVE_HEAD(head, field) do {				\
+	if ((STAILQ_FIRST((head)) =					\
+	     STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL)		\
+		(head)->stqh_last = &STAILQ_FIRST((head));		\
+} while (0)
+
+#define STAILQ_REMOVE_HEAD_UNTIL(head, elm, field) do {                 \
+       if ((STAILQ_FIRST((head)) = STAILQ_NEXT((elm), field)) == NULL) \
+               (head)->stqh_last = &STAILQ_FIRST((head));              \
+} while (0)
+
+#define STAILQ_REMOVE_AFTER(head, elm, field) do {			\
+	if ((STAILQ_NEXT(elm, field) =					\
+	     STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL)	\
+		(head)->stqh_last = &STAILQ_NEXT((elm), field);		\
+} while (0)
+
+#define STAILQ_SWAP(head1, head2, type) do {				\
+	struct type *swap_first = STAILQ_FIRST(head1);			\
+	struct type **swap_last = (head1)->stqh_last;			\
+	STAILQ_FIRST(head1) = STAILQ_FIRST(head2);			\
+	(head1)->stqh_last = (head2)->stqh_last;			\
+	STAILQ_FIRST(head2) = swap_first;				\
+	(head2)->stqh_last = swap_last;					\
+	if (STAILQ_EMPTY(head1))					\
+		(head1)->stqh_last = &STAILQ_FIRST(head1);		\
+	if (STAILQ_EMPTY(head2))					\
+		(head2)->stqh_last = &STAILQ_FIRST(head2);		\
+} while (0)
+
+
+/*
+ * List declarations.
+ */
+#define	LIST_HEAD(name, type)						\
+struct name {								\
+	struct type *lh_first;	/* first element */			\
+}
+
+#define	LIST_HEAD_INITIALIZER(head)					\
+	{ NULL }
+
+#define	LIST_ENTRY(type)						\
+struct {								\
+	struct type *le_next;	/* next element */			\
+	struct type **le_prev;	/* address of previous next element */	\
+}
+
+/*
+ * List functions.
+ */
+
+#if (defined(_KERNEL) && defined(INVARIANTS)) || defined(QUEUE_MACRO_DEBUG)
+#define	QMD_LIST_CHECK_HEAD(head, field) do {				\
+	if (LIST_FIRST((head)) != NULL &&				\
+	    LIST_FIRST((head))->field.le_prev !=			\
+	     &LIST_FIRST((head)))					\
+		panic("Bad list head %p first->prev != head", (head));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_NEXT(elm, field) do {				\
+	if (LIST_NEXT((elm), field) != NULL &&				\
+	    LIST_NEXT((elm), field)->field.le_prev !=			\
+	     &((elm)->field.le_next))					\
+	     	panic("Bad link elm %p next->prev != elm", (elm));	\
+} while (0)
+
+#define	QMD_LIST_CHECK_PREV(elm, field) do {				\
+	if (*(elm)->field.le_prev != (elm))				\
+		panic("Bad link elm %p prev->next != elm", (elm));	\
+} while (0)
+#else
+#define	QMD_LIST_CHECK_HEAD(head, field)
+#define	QMD_LIST_CHECK_NEXT(elm, field)
+#define	QMD_LIST_CHECK_PREV(elm, field)
+#endif /* (_KERNEL && INVARIANTS) || QUEUE_MACRO_DEBUG */
+
+#define	LIST_EMPTY(head)	((head)->lh_first == NULL)
+
+#define	LIST_FIRST(head)	((head)->lh_first)
+
+#define	LIST_FOREACH(var, head, field)					\
+	for ((var) = LIST_FIRST((head));				\
+	    (var);							\
+	    (var) = LIST_NEXT((var), field))
+
+#define	LIST_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = LIST_FIRST((head));				\
+	    (var) && ((tvar) = LIST_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	LIST_INIT(head) do {						\
+	LIST_FIRST((head)) = NULL;					\
+} while (0)
+
+#define	LIST_INSERT_AFTER(listelm, elm, field) do {			\
+	QMD_LIST_CHECK_NEXT(listelm, field);				\
+	if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\
+		LIST_NEXT((listelm), field)->field.le_prev =		\
+		    &LIST_NEXT((elm), field);				\
+	LIST_NEXT((listelm), field) = (elm);				\
+	(elm)->field.le_prev = &LIST_NEXT((listelm), field);		\
+} while (0)
+
+#define	LIST_INSERT_BEFORE(listelm, elm, field) do {			\
+	QMD_LIST_CHECK_PREV(listelm, field);				\
+	(elm)->field.le_prev = (listelm)->field.le_prev;		\
+	LIST_NEXT((elm), field) = (listelm);				\
+	*(listelm)->field.le_prev = (elm);				\
+	(listelm)->field.le_prev = &LIST_NEXT((elm), field);		\
+} while (0)
+
+#define	LIST_INSERT_HEAD(head, elm, field) do {				\
+	QMD_LIST_CHECK_HEAD((head), field);				\
+	if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL)	\
+		LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\
+	LIST_FIRST((head)) = (elm);					\
+	(elm)->field.le_prev = &LIST_FIRST((head));			\
+} while (0)
+
+#define	LIST_NEXT(elm, field)	((elm)->field.le_next)
+
+#define	LIST_REMOVE(elm, field) do {					\
+	QMD_LIST_CHECK_NEXT(elm, field);				\
+	QMD_LIST_CHECK_PREV(elm, field);				\
+	if (LIST_NEXT((elm), field) != NULL)				\
+		LIST_NEXT((elm), field)->field.le_prev = 		\
+		    (elm)->field.le_prev;				\
+	*(elm)->field.le_prev = LIST_NEXT((elm), field);		\
+	TRASHIT((elm)->field.le_next);					\
+	TRASHIT((elm)->field.le_prev);					\
+} while (0)
+
+#define LIST_SWAP(head1, head2, type, field) do {			\
+	struct type *swap_tmp = LIST_FIRST((head1));			\
+	LIST_FIRST((head1)) = LIST_FIRST((head2));			\
+	LIST_FIRST((head2)) = swap_tmp;					\
+	if ((swap_tmp = LIST_FIRST((head1))) != NULL)			\
+		swap_tmp->field.le_prev = &LIST_FIRST((head1));		\
+	if ((swap_tmp = LIST_FIRST((head2))) != NULL)			\
+		swap_tmp->field.le_prev = &LIST_FIRST((head2));		\
+} while (0)
+
+/*
+ * Tail queue declarations.
+ */
+#define	TAILQ_HEAD(name, type)						\
+struct name {								\
+	struct type *tqh_first;	/* first element */			\
+	struct type **tqh_last;	/* addr of last next element */		\
+	TRACEBUF							\
+}
+
+#define	TAILQ_HEAD_INITIALIZER(head)					\
+	{ NULL, &(head).tqh_first }
+
+#define	TAILQ_ENTRY(type)						\
+struct {								\
+	struct type *tqe_next;	/* next element */			\
+	struct type **tqe_prev;	/* address of previous next element */	\
+	TRACEBUF							\
+}
+
+/*
+ * Tail queue functions.
+ */
+#define	TAILQ_CONCAT(head1, head2, field) do {				\
+	if (!TAILQ_EMPTY(head2)) {					\
+		*(head1)->tqh_last = (head2)->tqh_first;		\
+		(head2)->tqh_first->field.tqe_prev = (head1)->tqh_last;	\
+		(head1)->tqh_last = (head2)->tqh_last;			\
+		TAILQ_INIT((head2));					\
+		QMD_TRACE_HEAD(head1);					\
+		QMD_TRACE_HEAD(head2);					\
+	}								\
+} while (0)
+
+#define	TAILQ_EMPTY(head)	((head)->tqh_first == NULL)
+
+#define	TAILQ_FIRST(head)	((head)->tqh_first)
+
+#define	TAILQ_FOREACH(var, head, field)					\
+	for ((var) = TAILQ_FIRST((head));				\
+	    (var);							\
+	    (var) = TAILQ_NEXT((var), field))
+
+#define	TAILQ_FOREACH_SAFE(var, head, field, tvar)			\
+	for ((var) = TAILQ_FIRST((head));				\
+	    (var) && ((tvar) = TAILQ_NEXT((var), field), 1);		\
+	    (var) = (tvar))
+
+#define	TAILQ_FOREACH_REVERSE(var, head, headname, field)		\
+	for ((var) = TAILQ_LAST((head), headname);			\
+	    (var);							\
+	    (var) = TAILQ_PREV((var), headname, field))
+
+#define	TAILQ_FOREACH_REVERSE_SAFE(var, head, headname, field, tvar)	\
+	for ((var) = TAILQ_LAST((head), headname);			\
+	    (var) && ((tvar) = TAILQ_PREV((var), headname, field), 1);	\
+	    (var) = (tvar))
+
+#define	TAILQ_INIT(head) do {						\
+	TAILQ_FIRST((head)) = NULL;					\
+	(head)->tqh_last = &TAILQ_FIRST((head));			\
+	QMD_TRACE_HEAD(head);						\
+} while (0)
+
+#define	TAILQ_INSERT_AFTER(head, listelm, elm, field) do {		\
+	if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\
+		TAILQ_NEXT((elm), field)->field.tqe_prev = 		\
+		    &TAILQ_NEXT((elm), field);				\
+	else {								\
+		(head)->tqh_last = &TAILQ_NEXT((elm), field);		\
+		QMD_TRACE_HEAD(head);					\
+	}								\
+	TAILQ_NEXT((listelm), field) = (elm);				\
+	(elm)->field.tqe_prev = &TAILQ_NEXT((listelm), field);		\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+	QMD_TRACE_ELEM(&listelm->field);				\
+} while (0)
+
+#define	TAILQ_INSERT_BEFORE(listelm, elm, field) do {			\
+	(elm)->field.tqe_prev = (listelm)->field.tqe_prev;		\
+	TAILQ_NEXT((elm), field) = (listelm);				\
+	*(listelm)->field.tqe_prev = (elm);				\
+	(listelm)->field.tqe_prev = &TAILQ_NEXT((elm), field);		\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+	QMD_TRACE_ELEM(&listelm->field);				\
+} while (0)
+
+#define	TAILQ_INSERT_HEAD(head, elm, field) do {			\
+	if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL)	\
+		TAILQ_FIRST((head))->field.tqe_prev =			\
+		    &TAILQ_NEXT((elm), field);				\
+	else								\
+		(head)->tqh_last = &TAILQ_NEXT((elm), field);		\
+	TAILQ_FIRST((head)) = (elm);					\
+	(elm)->field.tqe_prev = &TAILQ_FIRST((head));			\
+	QMD_TRACE_HEAD(head);						\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+} while (0)
+
+#define	TAILQ_INSERT_TAIL(head, elm, field) do {			\
+	TAILQ_NEXT((elm), field) = NULL;				\
+	(elm)->field.tqe_prev = (head)->tqh_last;			\
+	*(head)->tqh_last = (elm);					\
+	(head)->tqh_last = &TAILQ_NEXT((elm), field);			\
+	QMD_TRACE_HEAD(head);						\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+} while (0)
+
+#define	TAILQ_LAST(head, headname)					\
+	(*(((struct headname *)((head)->tqh_last))->tqh_last))
+
+#define	TAILQ_NEXT(elm, field) ((elm)->field.tqe_next)
+
+#define	TAILQ_PREV(elm, headname, field)				\
+	(*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
+
+#define	TAILQ_REMOVE(head, elm, field) do {				\
+	if ((TAILQ_NEXT((elm), field)) != NULL)				\
+		TAILQ_NEXT((elm), field)->field.tqe_prev = 		\
+		    (elm)->field.tqe_prev;				\
+	else {								\
+		(head)->tqh_last = (elm)->field.tqe_prev;		\
+		QMD_TRACE_HEAD(head);					\
+	}								\
+	*(elm)->field.tqe_prev = TAILQ_NEXT((elm), field);		\
+	TRASHIT((elm)->field.tqe_next);					\
+	TRASHIT((elm)->field.tqe_prev);					\
+	QMD_TRACE_ELEM(&(elm)->field);					\
+} while (0)
+
+#define TAILQ_SWAP(head1, head2, type, field) do {                      \
+	struct type *swap_first = (head1)->tqh_first;                   \
+	struct type **swap_last = (head1)->tqh_last;                    \
+	(head1)->tqh_first = (head2)->tqh_first;                        \
+	(head1)->tqh_last = (head2)->tqh_last;                          \
+	(head2)->tqh_first = swap_first;                                \
+	(head2)->tqh_last = swap_last;                                  \
+	if ((swap_first = (head1)->tqh_first) != NULL)                  \
+		swap_first->field.tqe_prev = &(head1)->tqh_first;       \
+	else                                                            \
+		(head1)->tqh_last = &(head1)->tqh_first;                \
+	if ((swap_first = (head2)->tqh_first) != NULL)                  \
+		swap_first->field.tqe_prev = &(head2)->tqh_first;       \
+	else                                                            \
+		(head2)->tqh_last = &(head2)->tqh_first;                \
+} while (0)
+
+/*
+ * Circular queue definitions.
+ */
+#define CIRCLEQ_HEAD(name, type)					\
+struct name {								\
+	struct type *cqh_first;		/* first element */		\
+	struct type *cqh_last;		/* last element */		\
+}
+
+#define CIRCLEQ_ENTRY(type)						\
+struct {								\
+	struct type *cqe_next;		/* next element */		\
+	struct type *cqe_prev;		/* previous element */		\
+}
+
+/*
+ * Circular queue functions.
+ */
+#define CIRCLEQ_EMPTY(head) ((head)->cqh_first == (void *)(head))
+
+#define CIRCLEQ_FIRST(head) ((head)->cqh_first)
+
+#define CIRCLEQ_FOREACH(var, head, field)				\
+	for((var) = (head)->cqh_first;					\
+	    (var) != (void *)(head);					\
+	    (var) = (var)->field.cqe_next)
+
+#define	CIRCLEQ_INIT(head) do {						\
+	(head)->cqh_first = (void *)(head);				\
+	(head)->cqh_last = (void *)(head);				\
+} while (0)
+
+#define CIRCLEQ_INSERT_AFTER(head, listelm, elm, field) do {		\
+	(elm)->field.cqe_next = (listelm)->field.cqe_next;		\
+	(elm)->field.cqe_prev = (listelm);				\
+	if ((listelm)->field.cqe_next == (void *)(head))		\
+		(head)->cqh_last = (elm);				\
+	else								\
+		(listelm)->field.cqe_next->field.cqe_prev = (elm);	\
+	(listelm)->field.cqe_next = (elm);				\
+} while (0)
+
+#define CIRCLEQ_INSERT_BEFORE(head, listelm, elm, field) do {		\
+	(elm)->field.cqe_next = (listelm);				\
+	(elm)->field.cqe_prev = (listelm)->field.cqe_prev;		\
+	if ((listelm)->field.cqe_prev == (void *)(head))		\
+		(head)->cqh_first = (elm);				\
+	else								\
+		(listelm)->field.cqe_prev->field.cqe_next = (elm);	\
+	(listelm)->field.cqe_prev = (elm);				\
+} while (0)
+
+#define CIRCLEQ_INSERT_HEAD(head, elm, field) do {			\
+	(elm)->field.cqe_next = (head)->cqh_first;			\
+	(elm)->field.cqe_prev = (void *)(head);				\
+	if ((head)->cqh_last == (void *)(head))				\
+		(head)->cqh_last = (elm);				\
+	else								\
+		(head)->cqh_first->field.cqe_prev = (elm);		\
+	(head)->cqh_first = (elm);					\
+} while (0)
+
+#define CIRCLEQ_INSERT_TAIL(head, elm, field) do {			\
+	(elm)->field.cqe_next = (void *)(head);				\
+	(elm)->field.cqe_prev = (head)->cqh_last;			\
+	if ((head)->cqh_first == (void *)(head))			\
+		(head)->cqh_first = (elm);				\
+	else								\
+		(head)->cqh_last->field.cqe_next = (elm);		\
+	(head)->cqh_last = (elm);					\
+} while (0)
+
+#define CIRCLEQ_LAST(head) ((head)->cqh_last)
+
+#define CIRCLEQ_NEXT(elm,field) ((elm)->field.cqe_next)
+
+#define CIRCLEQ_PREV(elm,field) ((elm)->field.cqe_prev)
+
+#define	CIRCLEQ_REMOVE(head, elm, field) do {				\
+	if ((elm)->field.cqe_next == (void *)(head))			\
+		(head)->cqh_last = (elm)->field.cqe_prev;		\
+	else								\
+		(elm)->field.cqe_next->field.cqe_prev =			\
+		    (elm)->field.cqe_prev;				\
+	if ((elm)->field.cqe_prev == (void *)(head))			\
+		(head)->cqh_first = (elm)->field.cqe_next;		\
+	else								\
+		(elm)->field.cqe_prev->field.cqe_next =			\
+		    (elm)->field.cqe_next;				\
+} while (0)
+
+#ifdef _KERNEL
+
+#if NOTFB31
+
+/*
+ * XXX insque() and remque() are an old way of handling certain queues.
+ * They bogusly assumes that all queue heads look alike.
+ */
+
+struct quehead {
+	struct quehead *qh_link;
+	struct quehead *qh_rlink;
+};
+
+#ifdef __GNUC__
+
+static __inline void
+insque(void *a, void *b)
+{
+	struct quehead *element = (struct quehead *)a,
+		 *head = (struct quehead *)b;
+
+	element->qh_link = head->qh_link;
+	element->qh_rlink = head;
+	head->qh_link = element;
+	element->qh_link->qh_rlink = element;
+}
+
+static __inline void
+remque(void *a)
+{
+	struct quehead *element = (struct quehead *)a;
+
+	element->qh_link->qh_rlink = element->qh_rlink;
+	element->qh_rlink->qh_link = element->qh_link;
+	element->qh_rlink = 0;
+}
+
+#else /* !__GNUC__ */
+
+void	insque(void *a, void *b);
+void	remque(void *a);
+
+#endif /* __GNUC__ */
+
+#endif
+#endif /* _KERNEL */
+
+#endif /* !_SYS_QUEUE_H_ */