如何使div仅从一小部分可拖动？

<div id="draggableDiv"></div>

#draggableDiv {
  width: 200px;
  height: 200px;
  background-color: #ccc;
  position: absolute;
  left: 100px;
  top: 100px;
}

var draggableDiv = document.getElementById("draggableDiv");
var offsetX, offsetY;

draggableDiv.addEventListener("mousedown", function(event) {
  offsetX = event.clientX - draggableDiv.offsetLeft;
  offsetY = event.clientY - draggableDiv.offsetTop;
  
  document.addEventListener("mousemove", moveDiv);
});

document.addEventListener("mouseup", function() {
  document.removeEventListener("mousemove", moveDiv);
});

function moveDiv(event) {
  draggableDiv.style.left = event.clientX - offsetX + "px";
  draggableDiv.style.top = event.clientY - offsetY + "px";
}