mr.c (d5ea2df9cefa9c81a66021b5bb89562d02bbc2f7) mr.c (3161625589c1d7c54e949d462f4d0c327664881a)
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 760 unchanged lines hidden (view full) ---

769 __be64 *pas;
770 struct device *ddev = dev->ib_dev.dma_device;
771
772 /*
773 * UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
774 * To avoid copying garbage after the pas array, we allocate
775 * a little more.
776 */
1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:

--- 760 unchanged lines hidden (view full) ---

769 __be64 *pas;
770 struct device *ddev = dev->ib_dev.dma_device;
771
772 /*
773 * UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
774 * To avoid copying garbage after the pas array, we allocate
775 * a little more.
776 */
777 *size = ALIGN(sizeof(u64) * npages, MLX5_UMR_MTT_ALIGNMENT);
777 *size = ALIGN(sizeof(struct mlx5_mtt) * npages, MLX5_UMR_MTT_ALIGNMENT);
778 *mr_pas = kmalloc(*size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
779 if (!(*mr_pas))
780 return -ENOMEM;
781
782 pas = PTR_ALIGN(*mr_pas, MLX5_UMR_ALIGN);
783 mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT);
784 /* Clear padding after the actual pages. */
778 *mr_pas = kmalloc(*size + MLX5_UMR_ALIGN - 1, GFP_KERNEL);
779 if (!(*mr_pas))
780 return -ENOMEM;
781
782 pas = PTR_ALIGN(*mr_pas, MLX5_UMR_ALIGN);
783 mlx5_ib_populate_pas(dev, umem, page_shift, pas, MLX5_IB_MTT_PRESENT);
784 /* Clear padding after the actual pages. */
785 memset(pas + npages, 0, *size - npages * sizeof(u64));
785 memset(pas + npages, 0, *size - npages * sizeof(struct mlx5_mtt));
786
787 *dma = dma_map_single(ddev, pas, *size, DMA_TO_DEVICE);
788 if (dma_mapping_error(ddev, *dma)) {
789 kfree(*mr_pas);
790 return -ENOMEM;
791 }
792
793 return 0;
794}
795
796static void prep_umr_wqe_common(struct ib_pd *pd, struct ib_send_wr *wr,
797 struct ib_sge *sg, u64 dma, int n, u32 key,
798 int page_shift)
799{
800 struct mlx5_ib_dev *dev = to_mdev(pd->device);
801 struct mlx5_umr_wr *umrwr = umr_wr(wr);
802
803 sg->addr = dma;
786
787 *dma = dma_map_single(ddev, pas, *size, DMA_TO_DEVICE);
788 if (dma_mapping_error(ddev, *dma)) {
789 kfree(*mr_pas);
790 return -ENOMEM;
791 }
792
793 return 0;
794}
795
796static void prep_umr_wqe_common(struct ib_pd *pd, struct ib_send_wr *wr,
797 struct ib_sge *sg, u64 dma, int n, u32 key,
798 int page_shift)
799{
800 struct mlx5_ib_dev *dev = to_mdev(pd->device);
801 struct mlx5_umr_wr *umrwr = umr_wr(wr);
802
803 sg->addr = dma;
804 sg->length = ALIGN(sizeof(u64) * n, 64);
804 sg->length = ALIGN(sizeof(struct mlx5_mtt) * n,
805 MLX5_IB_UMR_XLT_ALIGNMENT);
805 sg->lkey = dev->umrc.pd->local_dma_lkey;
806
807 wr->next = NULL;
808 wr->sg_list = sg;
809 if (n)
810 wr->num_sge = 1;
811 else
812 wr->num_sge = 0;
813
814 wr->opcode = MLX5_IB_WR_UMR;
815
806 sg->lkey = dev->umrc.pd->local_dma_lkey;
807
808 wr->next = NULL;
809 wr->sg_list = sg;
810 if (n)
811 wr->num_sge = 1;
812 else
813 wr->num_sge = 0;
814
815 wr->opcode = MLX5_IB_WR_UMR;
816
816 umrwr->npages = n;
817 umrwr->xlt_size = sg->length;
817 umrwr->page_shift = page_shift;
818 umrwr->mkey = key;
819}
820
821static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
822 struct ib_sge *sg, u64 dma, int n, u32 key,
823 int page_shift, u64 virt_addr, u64 len,
824 int access_flags)
825{
826 struct mlx5_umr_wr *umrwr = umr_wr(wr);
827
828 prep_umr_wqe_common(pd, wr, sg, dma, n, key, page_shift);
829
818 umrwr->page_shift = page_shift;
819 umrwr->mkey = key;
820}
821
822static void prep_umr_reg_wqe(struct ib_pd *pd, struct ib_send_wr *wr,
823 struct ib_sge *sg, u64 dma, int n, u32 key,
824 int page_shift, u64 virt_addr, u64 len,
825 int access_flags)
826{
827 struct mlx5_umr_wr *umrwr = umr_wr(wr);
828
829 prep_umr_wqe_common(pd, wr, sg, dma, n, key, page_shift);
830
830 wr->send_flags = 0;
831 wr->send_flags = MLX5_IB_SEND_UMR_ENABLE_MR |
832 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION |
833 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
831
834
832 umrwr->target.virt_addr = virt_addr;
835 umrwr->virt_addr = virt_addr;
833 umrwr->length = len;
834 umrwr->access_flags = access_flags;
835 umrwr->pd = pd;
836}
837
838static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
839 struct ib_send_wr *wr, u32 key)
840{
841 struct mlx5_umr_wr *umrwr = umr_wr(wr);
842
836 umrwr->length = len;
837 umrwr->access_flags = access_flags;
838 umrwr->pd = pd;
839}
840
841static void prep_umr_unreg_wqe(struct mlx5_ib_dev *dev,
842 struct ib_send_wr *wr, u32 key)
843{
844 struct mlx5_umr_wr *umrwr = umr_wr(wr);
845
843 wr->send_flags = MLX5_IB_SEND_UMR_UNREG | MLX5_IB_SEND_UMR_FAIL_IF_FREE;
846 wr->send_flags = MLX5_IB_SEND_UMR_DISABLE_MR |
847 MLX5_IB_SEND_UMR_FAIL_IF_FREE;
844 wr->opcode = MLX5_IB_WR_UMR;
845 umrwr->mkey = key;
846}
847
848static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
849 int access_flags, struct ib_umem **umem,
850 int *npages, int *page_shift, int *ncont,
851 int *order)

--- 136 unchanged lines hidden (view full) ---

988 struct device *ddev = dev->ib_dev.dma_device;
989 struct ib_umem *umem = mr->umem;
990 int size;
991 __be64 *pas;
992 dma_addr_t dma;
993 struct mlx5_umr_wr wr;
994 struct ib_sge sg;
995 int err = 0;
848 wr->opcode = MLX5_IB_WR_UMR;
849 umrwr->mkey = key;
850}
851
852static int mr_umem_get(struct ib_pd *pd, u64 start, u64 length,
853 int access_flags, struct ib_umem **umem,
854 int *npages, int *page_shift, int *ncont,
855 int *order)

--- 136 unchanged lines hidden (view full) ---

992 struct device *ddev = dev->ib_dev.dma_device;
993 struct ib_umem *umem = mr->umem;
994 int size;
995 __be64 *pas;
996 dma_addr_t dma;
997 struct mlx5_umr_wr wr;
998 struct ib_sge sg;
999 int err = 0;
996 const int page_index_alignment = MLX5_UMR_MTT_ALIGNMENT / sizeof(u64);
1000 const int page_index_alignment = MLX5_UMR_MTT_ALIGNMENT /
1001 sizeof(struct mlx5_mtt);
997 const int page_index_mask = page_index_alignment - 1;
998 size_t pages_mapped = 0;
999 size_t pages_to_map = 0;
1000 size_t pages_iter = 0;
1001 int use_emergency_buf = 0;
1002
1003 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
1004 * so we need to align the offset and length accordingly */
1005 if (start_page_index & page_index_mask) {
1006 npages += start_page_index & page_index_mask;
1007 start_page_index &= ~page_index_mask;
1008 }
1009
1010 pages_to_map = ALIGN(npages, page_index_alignment);
1011
1012 if (start_page_index + pages_to_map > MLX5_MAX_UMR_PAGES)
1013 return -EINVAL;
1014
1002 const int page_index_mask = page_index_alignment - 1;
1003 size_t pages_mapped = 0;
1004 size_t pages_to_map = 0;
1005 size_t pages_iter = 0;
1006 int use_emergency_buf = 0;
1007
1008 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes,
1009 * so we need to align the offset and length accordingly */
1010 if (start_page_index & page_index_mask) {
1011 npages += start_page_index & page_index_mask;
1012 start_page_index &= ~page_index_mask;
1013 }
1014
1015 pages_to_map = ALIGN(npages, page_index_alignment);
1016
1017 if (start_page_index + pages_to_map > MLX5_MAX_UMR_PAGES)
1018 return -EINVAL;
1019
1015 size = sizeof(u64) * pages_to_map;
1020 size = sizeof(struct mlx5_mtt) * pages_to_map;
1016 size = min_t(int, PAGE_SIZE, size);
1017 /* We allocate with GFP_ATOMIC to avoid recursion into page-reclaim
1018 * code, when we are called from an invalidation. The pas buffer must
1019 * be 2k-aligned for Connect-IB. */
1020 pas = (__be64 *)get_zeroed_page(GFP_ATOMIC);
1021 if (!pas) {
1022 mlx5_ib_warn(dev, "unable to allocate memory during MTT update, falling back to slower chunked mechanism.\n");
1023 pas = mlx5_ib_update_mtt_emergency_buffer;
1024 size = MLX5_UMR_MTT_MIN_CHUNK_SIZE;
1025 use_emergency_buf = 1;
1026 mutex_lock(&mlx5_ib_update_mtt_emergency_buffer_mutex);
1027 memset(pas, 0, size);
1028 }
1021 size = min_t(int, PAGE_SIZE, size);
1022 /* We allocate with GFP_ATOMIC to avoid recursion into page-reclaim
1023 * code, when we are called from an invalidation. The pas buffer must
1024 * be 2k-aligned for Connect-IB. */
1025 pas = (__be64 *)get_zeroed_page(GFP_ATOMIC);
1026 if (!pas) {
1027 mlx5_ib_warn(dev, "unable to allocate memory during MTT update, falling back to slower chunked mechanism.\n");
1028 pas = mlx5_ib_update_mtt_emergency_buffer;
1029 size = MLX5_UMR_MTT_MIN_CHUNK_SIZE;
1030 use_emergency_buf = 1;
1031 mutex_lock(&mlx5_ib_update_mtt_emergency_buffer_mutex);
1032 memset(pas, 0, size);
1033 }
1029 pages_iter = size / sizeof(u64);
1034 pages_iter = size / sizeof(struct mlx5_mtt);
1030 dma = dma_map_single(ddev, pas, size, DMA_TO_DEVICE);
1031 if (dma_mapping_error(ddev, dma)) {
1032 mlx5_ib_err(dev, "unable to map DMA during MTT update.\n");
1033 err = -ENOMEM;
1034 goto free_pas;
1035 }
1036
1037 for (pages_mapped = 0;

--- 6 unchanged lines hidden (view full) ---

1044 ib_umem_num_pages(umem) - start_page_index);
1045
1046 if (!zap) {
1047 __mlx5_ib_populate_pas(dev, umem, PAGE_SHIFT,
1048 start_page_index, npages, pas,
1049 MLX5_IB_MTT_PRESENT);
1050 /* Clear padding after the pages brought from the
1051 * umem. */
1035 dma = dma_map_single(ddev, pas, size, DMA_TO_DEVICE);
1036 if (dma_mapping_error(ddev, dma)) {
1037 mlx5_ib_err(dev, "unable to map DMA during MTT update.\n");
1038 err = -ENOMEM;
1039 goto free_pas;
1040 }
1041
1042 for (pages_mapped = 0;

--- 6 unchanged lines hidden (view full) ---

1049 ib_umem_num_pages(umem) - start_page_index);
1050
1051 if (!zap) {
1052 __mlx5_ib_populate_pas(dev, umem, PAGE_SHIFT,
1053 start_page_index, npages, pas,
1054 MLX5_IB_MTT_PRESENT);
1055 /* Clear padding after the pages brought from the
1056 * umem. */
1052 memset(pas + npages, 0, size - npages * sizeof(u64));
1057 memset(pas + npages, 0, size - npages *
1058 sizeof(struct mlx5_mtt));
1053 }
1054
1055 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1056
1057 memset(&wr, 0, sizeof(wr));
1058
1059 sg.addr = dma;
1059 }
1060
1061 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE);
1062
1063 memset(&wr, 0, sizeof(wr));
1064
1065 sg.addr = dma;
1060 sg.length = ALIGN(npages * sizeof(u64),
1066 sg.length = ALIGN(npages * sizeof(struct mlx5_mtt),
1061 MLX5_UMR_MTT_ALIGNMENT);
1062 sg.lkey = dev->umrc.pd->local_dma_lkey;
1063
1064 wr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE |
1067 MLX5_UMR_MTT_ALIGNMENT);
1068 sg.lkey = dev->umrc.pd->local_dma_lkey;
1069
1070 wr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE |
1065 MLX5_IB_SEND_UMR_UPDATE_MTT;
1071 MLX5_IB_SEND_UMR_UPDATE_XLT;
1066 wr.wr.sg_list = &sg;
1067 wr.wr.num_sge = 1;
1068 wr.wr.opcode = MLX5_IB_WR_UMR;
1072 wr.wr.sg_list = &sg;
1073 wr.wr.num_sge = 1;
1074 wr.wr.opcode = MLX5_IB_WR_UMR;
1069 wr.npages = sg.length / sizeof(u64);
1075 wr.xlt_size = sg.length;
1070 wr.page_shift = PAGE_SHIFT;
1071 wr.mkey = mr->mmkey.key;
1076 wr.page_shift = PAGE_SHIFT;
1077 wr.mkey = mr->mmkey.key;
1072 wr.target.offset = start_page_index;
1078 wr.offset = start_page_index * sizeof(struct mlx5_mtt);
1073
1074 err = mlx5_ib_post_send_wait(dev, &wr);
1075 }
1076 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1077
1078free_pas:
1079 if (!use_emergency_buf)
1080 free_page((unsigned long)pas);

--- 186 unchanged lines hidden (view full) ---

1267 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1268
1269 if (flags & IB_MR_REREG_TRANS) {
1270 err = dma_map_mr_pas(dev, mr->umem, npages, page_shift, &size,
1271 &mr_pas, &dma);
1272 if (err)
1273 return err;
1274
1079
1080 err = mlx5_ib_post_send_wait(dev, &wr);
1081 }
1082 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1083
1084free_pas:
1085 if (!use_emergency_buf)
1086 free_page((unsigned long)pas);

--- 186 unchanged lines hidden (view full) ---

1273 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE;
1274
1275 if (flags & IB_MR_REREG_TRANS) {
1276 err = dma_map_mr_pas(dev, mr->umem, npages, page_shift, &size,
1277 &mr_pas, &dma);
1278 if (err)
1279 return err;
1280
1275 umrwr.target.virt_addr = virt_addr;
1281 umrwr.virt_addr = virt_addr;
1276 umrwr.length = length;
1277 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1278 }
1279
1280 prep_umr_wqe_common(pd, &umrwr.wr, &sg, dma, npages, mr->mmkey.key,
1281 page_shift);
1282
1282 umrwr.length = length;
1283 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION;
1284 }
1285
1286 prep_umr_wqe_common(pd, &umrwr.wr, &sg, dma, npages, mr->mmkey.key,
1287 page_shift);
1288
1283 if (flags & IB_MR_REREG_PD) {
1289 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) {
1284 umrwr.pd = pd;
1290 umrwr.pd = pd;
1285 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD;
1286 }
1287
1288 if (flags & IB_MR_REREG_ACCESS) {
1289 umrwr.access_flags = access_flags;
1291 umrwr.access_flags = access_flags;
1290 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_ACCESS;
1292 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS;
1291 }
1292
1293 /* post send request to UMR QP */
1294 err = mlx5_ib_post_send_wait(dev, &umrwr);
1295
1296 if (flags & IB_MR_REREG_TRANS) {
1297 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1298 kfree(mr_pas);

--- 248 unchanged lines hidden (view full) ---

1547 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1548 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1549 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1550
1551 if (mr_type == IB_MR_TYPE_MEM_REG) {
1552 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1553 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
1554 err = mlx5_alloc_priv_descs(pd->device, mr,
1293 }
1294
1295 /* post send request to UMR QP */
1296 err = mlx5_ib_post_send_wait(dev, &umrwr);
1297
1298 if (flags & IB_MR_REREG_TRANS) {
1299 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE);
1300 kfree(mr_pas);

--- 248 unchanged lines hidden (view full) ---

1549 MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
1550 MLX5_SET(mkc, mkc, qpn, 0xffffff);
1551 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
1552
1553 if (mr_type == IB_MR_TYPE_MEM_REG) {
1554 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1555 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
1556 err = mlx5_alloc_priv_descs(pd->device, mr,
1555 ndescs, sizeof(u64));
1557 ndescs, sizeof(struct mlx5_mtt));
1556 if (err)
1557 goto err_free_in;
1558
1558 if (err)
1559 goto err_free_in;
1560
1559 mr->desc_size = sizeof(u64);
1561 mr->desc_size = sizeof(struct mlx5_mtt);
1560 mr->max_descs = ndescs;
1561 } else if (mr_type == IB_MR_TYPE_SG_GAPS) {
1562 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
1563
1564 err = mlx5_alloc_priv_descs(pd->device, mr,
1565 ndescs, sizeof(struct mlx5_klm));
1566 if (err)
1567 goto err_free_in;

--- 265 unchanged lines hidden --- 		1562 mr->max_descs = ndescs;
1563 } else if (mr_type == IB_MR_TYPE_SG_GAPS) {
1564 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
1565
1566 err = mlx5_alloc_priv_descs(pd->device, mr,
1567 ndescs, sizeof(struct mlx5_klm));
1568 if (err)
1569 goto err_free_in;

--- 265 unchanged lines hidden ---